eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from utils.infer import *\n",
    "import pandas as pd\n",
    "from  utils.eval import *\n",
    "from utils.chat import *\n",
    "from utils.data import *\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combining inferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "inference_paths = \"../data/inference\"\n",
    "output_path = \"../data/combined_inferences.jsonl\"\n",
    "\n",
    "combined = combine(inference_paths, output_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building evaluation inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_models = MODELS_TO_MODE.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "build_evaluation_inputs(output_path, models = all_models)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  [OLD] Evaluation GPT 3 direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gpt_3_eval_res = note_evaluation('direct-gpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>model_name</th>\n",
       "      <th>bleu</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_score</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>767</th>\n",
       "      <td>56341</td>\n",
       "      <td>A 46-year-old woman presented with intermitten...</td>\n",
       "      <td>Patient presented with a two-month history of ...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.384352</td>\n",
       "      <td>0.947115</td>\n",
       "      <td>8</td>\n",
       "      <td>0.938389</td>\n",
       "      <td>0.852381</td>\n",
       "      <td>0.928910</td>\n",
       "      <td>0.928910</td>\n",
       "      <td>1.406937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>125854</td>\n",
       "      <td>A healthy, athletic 19-year-old African Americ...</td>\n",
       "      <td>Patient presented for a surgical evaluation of...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.256832</td>\n",
       "      <td>0.885681</td>\n",
       "      <td>9</td>\n",
       "      <td>0.904645</td>\n",
       "      <td>0.796069</td>\n",
       "      <td>0.860636</td>\n",
       "      <td>0.860636</td>\n",
       "      <td>1.183339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>165122</td>\n",
       "      <td>A 19-year-old, single, unemployed, Caucasian f...</td>\n",
       "      <td>Patient presented with multiple pigmented annu...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.309861</td>\n",
       "      <td>0.901271</td>\n",
       "      <td>9</td>\n",
       "      <td>0.864450</td>\n",
       "      <td>0.771208</td>\n",
       "      <td>0.843990</td>\n",
       "      <td>0.843990</td>\n",
       "      <td>1.166625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534</th>\n",
       "      <td>146809</td>\n",
       "      <td>The present case is about a 48-year-old, femal...</td>\n",
       "      <td>Patient presented with complaints of fullness ...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.598819</td>\n",
       "      <td>0.882541</td>\n",
       "      <td>9</td>\n",
       "      <td>0.806604</td>\n",
       "      <td>0.658768</td>\n",
       "      <td>0.764151</td>\n",
       "      <td>0.764151</td>\n",
       "      <td>1.086166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>117607</td>\n",
       "      <td>The author reports a case of 30-year-old male ...</td>\n",
       "      <td>Patient presents to the physiotherapy outpatie...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.140051</td>\n",
       "      <td>0.879066</td>\n",
       "      <td>9</td>\n",
       "      <td>0.840295</td>\n",
       "      <td>0.735802</td>\n",
       "      <td>0.830467</td>\n",
       "      <td>0.830467</td>\n",
       "      <td>1.056809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>170283</td>\n",
       "      <td>A 70-year-old nonverbal female presented to th...</td>\n",
       "      <td>Patient presented to the emergency department ...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.182045</td>\n",
       "      <td>0.809504</td>\n",
       "      <td>8</td>\n",
       "      <td>0.340580</td>\n",
       "      <td>0.167883</td>\n",
       "      <td>0.231884</td>\n",
       "      <td>0.231884</td>\n",
       "      <td>0.229316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>163372</td>\n",
       "      <td>A 23-year-old male patient presented to us in ...</td>\n",
       "      <td>Patient presented with a deformity of the back...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.050793</td>\n",
       "      <td>0.915659</td>\n",
       "      <td>2</td>\n",
       "      <td>0.395480</td>\n",
       "      <td>0.187500</td>\n",
       "      <td>0.305085</td>\n",
       "      <td>0.305085</td>\n",
       "      <td>0.221921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>162274</td>\n",
       "      <td>A 26-year-old man, educated in an engineering ...</td>\n",
       "      <td>Patient presented with a chief complaint of co...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.165997</td>\n",
       "      <td>0.851599</td>\n",
       "      <td>7</td>\n",
       "      <td>0.347561</td>\n",
       "      <td>0.085890</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.218151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755</th>\n",
       "      <td>15191</td>\n",
       "      <td>A 55-year-old male who had been diagnosed with...</td>\n",
       "      <td>Patient presented with a two-week history of i...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.237331</td>\n",
       "      <td>0.890861</td>\n",
       "      <td>2</td>\n",
       "      <td>0.385852</td>\n",
       "      <td>0.213592</td>\n",
       "      <td>0.327974</td>\n",
       "      <td>0.327974</td>\n",
       "      <td>0.214040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>72695</td>\n",
       "      <td>We describe the case of a 47-year-old male who...</td>\n",
       "      <td>Mr. X presented to the clinic with a history o...</td>\n",
       "      <td>direct-gpt</td>\n",
       "      <td>0.107283</td>\n",
       "      <td>0.804507</td>\n",
       "      <td>8</td>\n",
       "      <td>0.205128</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.119658</td>\n",
       "      <td>0.119658</td>\n",
       "      <td>0.174135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "767   56341  A 46-year-old woman presented with intermitten...   \n",
       "307  125854  A healthy, athletic 19-year-old African Americ...   \n",
       "213  165122  A 19-year-old, single, unemployed, Caucasian f...   \n",
       "534  146809  The present case is about a 48-year-old, femal...   \n",
       "417  117607  The author reports a case of 30-year-old male ...   \n",
       "..      ...                                                ...   \n",
       "518  170283  A 70-year-old nonverbal female presented to th...   \n",
       "341  163372  A 23-year-old male patient presented to us in ...   \n",
       "270  162274  A 26-year-old man, educated in an engineering ...   \n",
       "755   15191  A 55-year-old male who had been diagnosed with...   \n",
       "479   72695  We describe the case of a 47-year-old male who...   \n",
       "\n",
       "                                                  pred  model_name      bleu  \\\n",
       "767  Patient presented with a two-month history of ...  direct-gpt  0.384352   \n",
       "307  Patient presented for a surgical evaluation of...  direct-gpt  0.256832   \n",
       "213  Patient presented with multiple pigmented annu...  direct-gpt  0.309861   \n",
       "534  Patient presented with complaints of fullness ...  direct-gpt  0.598819   \n",
       "417  Patient presents to the physiotherapy outpatie...  direct-gpt  0.140051   \n",
       "..                                                 ...         ...       ...   \n",
       "518  Patient presented to the emergency department ...  direct-gpt  0.182045   \n",
       "341  Patient presented with a deformity of the back...  direct-gpt  0.050793   \n",
       "270  Patient presented with a chief complaint of co...  direct-gpt  0.165997   \n",
       "755  Patient presented with a two-week history of i...  direct-gpt  0.237331   \n",
       "479  Mr. X presented to the clinic with a history o...  direct-gpt  0.107283   \n",
       "\n",
       "         bert  gpt_score    rouge1    rouge2    rougeL  rougeLsum  \\\n",
       "767  0.947115          8  0.938389  0.852381  0.928910   0.928910   \n",
       "307  0.885681          9  0.904645  0.796069  0.860636   0.860636   \n",
       "213  0.901271          9  0.864450  0.771208  0.843990   0.843990   \n",
       "534  0.882541          9  0.806604  0.658768  0.764151   0.764151   \n",
       "417  0.879066          9  0.840295  0.735802  0.830467   0.830467   \n",
       "..        ...        ...       ...       ...       ...        ...   \n",
       "518  0.809504          8  0.340580  0.167883  0.231884   0.231884   \n",
       "341  0.915659          2  0.395480  0.187500  0.305085   0.305085   \n",
       "270  0.851599          7  0.347561  0.085890  0.250000   0.250000   \n",
       "755  0.890861          2  0.385852  0.213592  0.327974   0.327974   \n",
       "479  0.804507          8  0.205128  0.017241  0.119658   0.119658   \n",
       "\n",
       "     aggregated_score  \n",
       "767          1.406937  \n",
       "307          1.183339  \n",
       "213          1.166625  \n",
       "534          1.086166  \n",
       "417          1.056809  \n",
       "..                ...  \n",
       "518          0.229316  \n",
       "341          0.221921  \n",
       "270          0.218151  \n",
       "755          0.214040  \n",
       "479          0.174135  \n",
       "\n",
       "[1000 rows x 12 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(gpt_3_eval_res.sort_values(by='aggregated_score', ascending=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-7b-summarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running summary evaluation with score types: ['bleu', 'rouge', 'bert', 'gpt_similarity_score']\n",
      "Loading data...\n",
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'flatten and match dicts': 'done', 'clean dicts and counts': 'done', 'summary_statistics': 'done', 'scores': 'done', 'pairs_idx': 'done', 'eval_by_sample': 'in progress', 'eval_by_key': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n"
     ]
    }
   ],
   "source": [
    "summarizer7_eval_by_sample , summarizer7_eval_by_key = summary_evaluation('meditron-7b-summarizer')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-13b-summarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running summary evaluation with score types: ['bleu', 'rouge', 'bert', 'gpt_similarity_score']\n",
      "Loading data...\n",
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'flatten and match dicts': 'done', 'clean dicts and counts': 'done', 'summary_statistics': 'done', 'scores': 'done', 'pairs_idx': 'done', 'eval_by_sample': 'in progress', 'eval_by_key': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n"
     ]
    }
   ],
   "source": [
    "summarizer13_eval_by_sample , summarizer13_eval_by_key = summary_evaluation('meditron-13b-summarizer')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merging summaries eval (and computing aggragated scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>mean_bleu</th>\n",
       "      <th>mean_bert</th>\n",
       "      <th>mean_gpt_similarity_score</th>\n",
       "      <th>mean_rouge1</th>\n",
       "      <th>mean_rouge2</th>\n",
       "      <th>mean_rougeL</th>\n",
       "      <th>mean_rougeLsum</th>\n",
       "      <th>aggregated_score</th>\n",
       "      <th>missing_keys_count_prop</th>\n",
       "      <th>extra_keys_count_prop</th>\n",
       "      <th>common_none_count_prop</th>\n",
       "      <th>gold_none_count_prop</th>\n",
       "      <th>pred_none_count_prop</th>\n",
       "      <th>common_non_none_prop</th>\n",
       "      <th>mean_number_of_keys</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meditron-7b-summarizer</td>\n",
       "      <td>0.208256</td>\n",
       "      <td>0.903149</td>\n",
       "      <td>7.356083</td>\n",
       "      <td>0.648584</td>\n",
       "      <td>0.461197</td>\n",
       "      <td>0.639717</td>\n",
       "      <td>0.639717</td>\n",
       "      <td>0.547134</td>\n",
       "      <td>0.164558</td>\n",
       "      <td>0.009932</td>\n",
       "      <td>0.429649</td>\n",
       "      <td>0.027024</td>\n",
       "      <td>0.051360</td>\n",
       "      <td>0.317477</td>\n",
       "      <td>80.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meditron-13b-summarizer</td>\n",
       "      <td>0.203400</td>\n",
       "      <td>0.902464</td>\n",
       "      <td>7.318198</td>\n",
       "      <td>0.644532</td>\n",
       "      <td>0.457226</td>\n",
       "      <td>0.635784</td>\n",
       "      <td>0.635784</td>\n",
       "      <td>0.543132</td>\n",
       "      <td>0.164570</td>\n",
       "      <td>0.009319</td>\n",
       "      <td>0.428958</td>\n",
       "      <td>0.027312</td>\n",
       "      <td>0.054123</td>\n",
       "      <td>0.315718</td>\n",
       "      <td>80.188</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     model  mean_bleu  mean_bert  mean_gpt_similarity_score  \\\n",
       "0   meditron-7b-summarizer   0.208256   0.903149                   7.356083   \n",
       "1  meditron-13b-summarizer   0.203400   0.902464                   7.318198   \n",
       "\n",
       "   mean_rouge1  mean_rouge2  mean_rougeL  mean_rougeLsum  aggregated_score  \\\n",
       "0     0.648584     0.461197     0.639717        0.639717          0.547134   \n",
       "1     0.644532     0.457226     0.635784        0.635784          0.543132   \n",
       "\n",
       "   missing_keys_count_prop  extra_keys_count_prop  common_none_count_prop  \\\n",
       "0                 0.164558               0.009932                0.429649   \n",
       "1                 0.164570               0.009319                0.428958   \n",
       "\n",
       "   gold_none_count_prop  pred_none_count_prop  common_non_none_prop  \\\n",
       "0              0.027024              0.051360              0.317477   \n",
       "1              0.027312              0.054123              0.315718   \n",
       "\n",
       "   mean_number_of_keys  \n",
       "0               80.201  \n",
       "1               80.188  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "final_summary_eval = merge_summary_evaluation(summarizer_models)\n",
    "display(final_summary_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>mean_bleu</th>\n",
       "      <th>mean_bert</th>\n",
       "      <th>mean_gpt_similarity_score</th>\n",
       "      <th>mean_rouge1</th>\n",
       "      <th>mean_rouge2</th>\n",
       "      <th>mean_rougeL</th>\n",
       "      <th>mean_rougeLsum</th>\n",
       "      <th>missing_keys_count</th>\n",
       "      <th>extra_keys_count</th>\n",
       "      <th>common_none_count</th>\n",
       "      <th>gold_none_count</th>\n",
       "      <th>pred_none_count</th>\n",
       "      <th>common_non_none</th>\n",
       "      <th>all_keys_count</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>171553</td>\n",
       "      <td>{'visit motivation': 'Episodes of continuous a...</td>\n",
       "      <td>{'visit motivation': 'Episodes of continuous a...</td>\n",
       "      <td>Doctor: Good morning, Mr. Smith. I see that yo...</td>\n",
       "      <td>0.485441</td>\n",
       "      <td>0.998456</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>0.993852</td>\n",
       "      <td>0.826977</td>\n",
       "      <td>0.993852</td>\n",
       "      <td>0.993852</td>\n",
       "      <td>34</td>\n",
       "      <td>6</td>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>83</td>\n",
       "      <td>0.860968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>884</th>\n",
       "      <td>196044</td>\n",
       "      <td>{'visit motivation': 'Inability to flex left e...</td>\n",
       "      <td>{'visit motivation': 'Inability to flex left e...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>0.612640</td>\n",
       "      <td>0.967639</td>\n",
       "      <td>9.480000</td>\n",
       "      <td>0.911776</td>\n",
       "      <td>0.811710</td>\n",
       "      <td>0.901959</td>\n",
       "      <td>0.901959</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>25</td>\n",
       "      <td>64</td>\n",
       "      <td>0.844289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>199095</td>\n",
       "      <td>{'visit motivation': 'Complaining of back pain...</td>\n",
       "      <td>{'visit motivation': 'Complaining of back pain...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>0.461334</td>\n",
       "      <td>0.989777</td>\n",
       "      <td>7.153846</td>\n",
       "      <td>0.945359</td>\n",
       "      <td>0.830744</td>\n",
       "      <td>0.939692</td>\n",
       "      <td>0.939692</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>39</td>\n",
       "      <td>76</td>\n",
       "      <td>0.837637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>598</th>\n",
       "      <td>197444</td>\n",
       "      <td>{'visit motivation': 'appetite loss', 'admissi...</td>\n",
       "      <td>{'visit motivation': 'appetite loss', 'admissi...</td>\n",
       "      <td>Doctor: Good morning, you were admitted to our...</td>\n",
       "      <td>0.574944</td>\n",
       "      <td>0.979784</td>\n",
       "      <td>7.562500</td>\n",
       "      <td>0.919436</td>\n",
       "      <td>0.771693</td>\n",
       "      <td>0.915963</td>\n",
       "      <td>0.915963</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>16</td>\n",
       "      <td>58</td>\n",
       "      <td>0.827107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>196330</td>\n",
       "      <td>{'visit motivation': 'Sudden increase in the i...</td>\n",
       "      <td>{'visit motivation': 'Sudden increase in the i...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>0.467932</td>\n",
       "      <td>0.983886</td>\n",
       "      <td>9.366667</td>\n",
       "      <td>0.928611</td>\n",
       "      <td>0.759967</td>\n",
       "      <td>0.928611</td>\n",
       "      <td>0.928611</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>30</td>\n",
       "      <td>82</td>\n",
       "      <td>0.823643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>121545</td>\n",
       "      <td>{'visit motivation': 'Complaints of vomiting a...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hi, what brings you here today?\\nPatie...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>96</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>37048</td>\n",
       "      <td>{'visit motivation': 'Pain, hemorrhage and a g...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, I'm Dr. Smith. How are you feel...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>67</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>67</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>2397</td>\n",
       "      <td>{'visit motivation': 'Pain on the lower right ...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, what brings you here today?\\nPa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>123492</td>\n",
       "      <td>{'visit motivation': 'Dizziness and near synco...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Good morning, what brings you to the e...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>83</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>83</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>52461</td>\n",
       "      <td>{'visit motivation': 'hypersexuality after sta...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "506  171553  {'visit motivation': 'Episodes of continuous a...   \n",
       "884  196044  {'visit motivation': 'Inability to flex left e...   \n",
       "444  199095  {'visit motivation': 'Complaining of back pain...   \n",
       "598  197444  {'visit motivation': 'appetite loss', 'admissi...   \n",
       "729  196330  {'visit motivation': 'Sudden increase in the i...   \n",
       "..      ...                                                ...   \n",
       "577  121545  {'visit motivation': 'Complaints of vomiting a...   \n",
       "581   37048  {'visit motivation': 'Pain, hemorrhage and a g...   \n",
       "587    2397  {'visit motivation': 'Pain on the lower right ...   \n",
       "617  123492  {'visit motivation': 'Dizziness and near synco...   \n",
       "805   52461  {'visit motivation': 'hypersexuality after sta...   \n",
       "\n",
       "                                                  pred  \\\n",
       "506  {'visit motivation': 'Episodes of continuous a...   \n",
       "884  {'visit motivation': 'Inability to flex left e...   \n",
       "444  {'visit motivation': 'Complaining of back pain...   \n",
       "598  {'visit motivation': 'appetite loss', 'admissi...   \n",
       "729  {'visit motivation': 'Sudden increase in the i...   \n",
       "..                                                 ...   \n",
       "577                                                 {}   \n",
       "581                                                 {}   \n",
       "587                                                 {}   \n",
       "617                                                 {}   \n",
       "805                                                 {}   \n",
       "\n",
       "                                          conversation  mean_bleu  mean_bert  \\\n",
       "506  Doctor: Good morning, Mr. Smith. I see that yo...   0.485441   0.998456   \n",
       "884  Doctor: Good morning, how are you feeling toda...   0.612640   0.967639   \n",
       "444  Doctor: Hello, how are you feeling today?\\nPat...   0.461334   0.989777   \n",
       "598  Doctor: Good morning, you were admitted to our...   0.574944   0.979784   \n",
       "729  Doctor: Hello, how are you feeling today?\\nPat...   0.467932   0.983886   \n",
       "..                                                 ...        ...        ...   \n",
       "577  Doctor: Hi, what brings you here today?\\nPatie...        NaN        NaN   \n",
       "581  Doctor: Hello, I'm Dr. Smith. How are you feel...        NaN        NaN   \n",
       "587  Doctor: Hello, what brings you here today?\\nPa...        NaN        NaN   \n",
       "617  Doctor: Good morning, what brings you to the e...        NaN        NaN   \n",
       "805  Doctor: Hello, how are you feeling today?\\nPat...        NaN        NaN   \n",
       "\n",
       "     mean_gpt_similarity_score  mean_rouge1  mean_rouge2  mean_rougeL  \\\n",
       "506                   5.500000     0.993852     0.826977     0.993852   \n",
       "884                   9.480000     0.911776     0.811710     0.901959   \n",
       "444                   7.153846     0.945359     0.830744     0.939692   \n",
       "598                   7.562500     0.919436     0.771693     0.915963   \n",
       "729                   9.366667     0.928611     0.759967     0.928611   \n",
       "..                         ...          ...          ...          ...   \n",
       "577                        NaN          NaN          NaN          NaN   \n",
       "581                        NaN          NaN          NaN          NaN   \n",
       "587                        NaN          NaN          NaN          NaN   \n",
       "617                        NaN          NaN          NaN          NaN   \n",
       "805                        NaN          NaN          NaN          NaN   \n",
       "\n",
       "     mean_rougeLsum  missing_keys_count  extra_keys_count  common_none_count  \\\n",
       "506        0.993852                  34                 6                 19   \n",
       "884        0.901959                   0                 0                 34   \n",
       "444        0.939692                   0                 0                 33   \n",
       "598        0.915963                   3                 0                 31   \n",
       "729        0.928611                  11                 0                 36   \n",
       "..              ...                 ...               ...                ...   \n",
       "577             NaN                  96                 0                  0   \n",
       "581             NaN                  67                 0                  0   \n",
       "587             NaN                  61                 0                  0   \n",
       "617             NaN                  83                 0                  0   \n",
       "805             NaN                  58                 0                  0   \n",
       "\n",
       "     gold_none_count  pred_none_count  common_non_none  all_keys_count  \\\n",
       "506                0                0               24              83   \n",
       "884                0                5               25              64   \n",
       "444                2                2               39              76   \n",
       "598                2                6               16              58   \n",
       "729                2                3               30              82   \n",
       "..               ...              ...              ...             ...   \n",
       "577                0                0                0              96   \n",
       "581                0                0                0              67   \n",
       "587                0                0                0              61   \n",
       "617                0                0                0              83   \n",
       "805                0                0                0              58   \n",
       "\n",
       "     aggregated_score  \n",
       "506          0.860968  \n",
       "884          0.844289  \n",
       "444          0.837637  \n",
       "598          0.827107  \n",
       "729          0.823643  \n",
       "..                ...  \n",
       "577          0.000000  \n",
       "581          0.000000  \n",
       "587          0.000000  \n",
       "617          0.000000  \n",
       "805          0.000000  \n",
       "\n",
       "[1000 rows x 19 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>keys</th>\n",
       "      <th>bleu</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>gold_none_prop</th>\n",
       "      <th>pred_none_prop</th>\n",
       "      <th>missing_keys_prop</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>admission/0/care center details</td>\n",
       "      <td>0.270086</td>\n",
       "      <td>0.918273</td>\n",
       "      <td>7.611111</td>\n",
       "      <td>0.735385</td>\n",
       "      <td>0.650588</td>\n",
       "      <td>0.726991</td>\n",
       "      <td>0.726991</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.153</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.510118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>admission/0/date</td>\n",
       "      <td>0.231097</td>\n",
       "      <td>0.919139</td>\n",
       "      <td>7.827957</td>\n",
       "      <td>0.761162</td>\n",
       "      <td>0.608599</td>\n",
       "      <td>0.754994</td>\n",
       "      <td>0.754994</td>\n",
       "      <td>0.020</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.565423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>admission/0/duration</td>\n",
       "      <td>0.116882</td>\n",
       "      <td>0.927714</td>\n",
       "      <td>8.325843</td>\n",
       "      <td>0.762572</td>\n",
       "      <td>0.690028</td>\n",
       "      <td>0.762572</td>\n",
       "      <td>0.762572</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.049</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.602280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>admission/0/reason</td>\n",
       "      <td>0.262550</td>\n",
       "      <td>0.884549</td>\n",
       "      <td>7.142216</td>\n",
       "      <td>0.552512</td>\n",
       "      <td>0.429219</td>\n",
       "      <td>0.531419</td>\n",
       "      <td>0.531419</td>\n",
       "      <td>0.038</td>\n",
       "      <td>0.069</td>\n",
       "      <td>0.012</td>\n",
       "      <td>0.429594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>admission/1/care center details</td>\n",
       "      <td>0.082227</td>\n",
       "      <td>0.941237</td>\n",
       "      <td>8.200000</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>0.795455</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.010</td>\n",
       "      <td>0.653332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>treatments/5/reaction to treatment</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.926654</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.461538</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.451646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>treatments/5/reason for taking</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.809110</td>\n",
       "      <td>4.666667</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.389713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>treatments/5/related condition</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.833170</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.308948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>226</th>\n",
       "      <td>treatments/5/time</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.659807</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.020751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>visit motivation</td>\n",
       "      <td>0.339092</td>\n",
       "      <td>0.910768</td>\n",
       "      <td>7.413472</td>\n",
       "      <td>0.677316</td>\n",
       "      <td>0.546617</td>\n",
       "      <td>0.653465</td>\n",
       "      <td>0.653465</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.008</td>\n",
       "      <td>0.584680</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>228 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   keys      bleu      bert  \\\n",
       "0       admission/0/care center details  0.270086  0.918273   \n",
       "1                      admission/0/date  0.231097  0.919139   \n",
       "2                  admission/0/duration  0.116882  0.927714   \n",
       "3                    admission/0/reason  0.262550  0.884549   \n",
       "4       admission/1/care center details  0.082227  0.941237   \n",
       "..                                  ...       ...       ...   \n",
       "223  treatments/5/reaction to treatment  0.000000  0.926654   \n",
       "224      treatments/5/reason for taking  0.333333  0.809110   \n",
       "225      treatments/5/related condition  0.000000  0.833170   \n",
       "226                   treatments/5/time  0.000000  0.659807   \n",
       "227                    visit motivation  0.339092  0.910768   \n",
       "\n",
       "     gpt_similarity_score    rouge1    rouge2    rougeL  rougeLsum  \\\n",
       "0                7.611111  0.735385  0.650588  0.726991   0.726991   \n",
       "1                7.827957  0.761162  0.608599  0.754994   0.754994   \n",
       "2                8.325843  0.762572  0.690028  0.762572   0.762572   \n",
       "3                7.142216  0.552512  0.429219  0.531419   0.531419   \n",
       "4                8.200000  0.795455  0.666667  0.795455   0.795455   \n",
       "..                    ...       ...       ...       ...        ...   \n",
       "223              8.000000  0.533333  0.461538  0.533333   0.533333   \n",
       "224              4.666667  0.416667  0.333333  0.416667   0.416667   \n",
       "225              5.500000  0.500000  0.000000  0.500000   0.500000   \n",
       "226              1.000000  0.000000  0.000000  0.000000   0.000000   \n",
       "227              7.413472  0.677316  0.546617  0.653465   0.653465   \n",
       "\n",
       "     gold_none_prop  pred_none_prop  missing_keys_prop  aggregated_score  \n",
       "0             0.048           0.153              0.012          0.510118  \n",
       "1             0.020           0.095              0.012          0.565423  \n",
       "2             0.021           0.049              0.012          0.602280  \n",
       "3             0.038           0.069              0.012          0.429594  \n",
       "4             0.001           0.003              0.010          0.653332  \n",
       "..              ...             ...                ...               ...  \n",
       "223           0.000           0.000              0.016          0.451646  \n",
       "224           0.000           0.000              0.016          0.389713  \n",
       "225           0.000           0.000              0.016          0.308948  \n",
       "226           0.000           0.000              0.016          0.020751  \n",
       "227           0.005           0.005              0.008          0.584680  \n",
       "\n",
       "[228 rows x 12 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>mean_bleu</th>\n",
       "      <th>mean_bert</th>\n",
       "      <th>mean_gpt_similarity_score</th>\n",
       "      <th>mean_rouge1</th>\n",
       "      <th>mean_rouge2</th>\n",
       "      <th>mean_rougeL</th>\n",
       "      <th>mean_rougeLsum</th>\n",
       "      <th>missing_keys_count</th>\n",
       "      <th>extra_keys_count</th>\n",
       "      <th>common_none_count</th>\n",
       "      <th>gold_none_count</th>\n",
       "      <th>pred_none_count</th>\n",
       "      <th>common_non_none</th>\n",
       "      <th>all_keys_count</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>171553</td>\n",
       "      <td>{'visit motivation': 'Episodes of continuous a...</td>\n",
       "      <td>{'visit motivation': 'Episodes of continuous a...</td>\n",
       "      <td>Doctor: Good morning, Mr. Smith. I see that yo...</td>\n",
       "      <td>0.418091</td>\n",
       "      <td>0.980467</td>\n",
       "      <td>9.333333</td>\n",
       "      <td>0.906004</td>\n",
       "      <td>0.767392</td>\n",
       "      <td>0.903359</td>\n",
       "      <td>0.903359</td>\n",
       "      <td>25</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>83</td>\n",
       "      <td>0.805180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>187015</td>\n",
       "      <td>{'visit motivation': 'Elective left and right ...</td>\n",
       "      <td>{'visit motivation': 'Elective left and right ...</td>\n",
       "      <td>Doctor: Good morning, Mr. Johnson. How are you...</td>\n",
       "      <td>0.463583</td>\n",
       "      <td>0.969643</td>\n",
       "      <td>9.520000</td>\n",
       "      <td>0.899577</td>\n",
       "      <td>0.700834</td>\n",
       "      <td>0.899577</td>\n",
       "      <td>0.899577</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>37</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>73</td>\n",
       "      <td>0.792847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>444</th>\n",
       "      <td>199095</td>\n",
       "      <td>{'visit motivation': 'Complaining of back pain...</td>\n",
       "      <td>{'visit motivation': 'Complaining of back pain...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>0.409682</td>\n",
       "      <td>0.979850</td>\n",
       "      <td>9.040000</td>\n",
       "      <td>0.908114</td>\n",
       "      <td>0.720753</td>\n",
       "      <td>0.899486</td>\n",
       "      <td>0.899486</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>25</td>\n",
       "      <td>76</td>\n",
       "      <td>0.789941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>676</th>\n",
       "      <td>183398</td>\n",
       "      <td>{'visit motivation': 'Complaining of atraumati...</td>\n",
       "      <td>{'visit motivation': 'Complaining of atraumati...</td>\n",
       "      <td>Doctor: Hi there, how are you feeling today?\\n...</td>\n",
       "      <td>0.264289</td>\n",
       "      <td>0.982445</td>\n",
       "      <td>9.291667</td>\n",
       "      <td>0.931441</td>\n",
       "      <td>0.735302</td>\n",
       "      <td>0.925330</td>\n",
       "      <td>0.925330</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>64</td>\n",
       "      <td>0.787908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>193172</td>\n",
       "      <td>{'visit motivation': 'Routine follow-up examin...</td>\n",
       "      <td>{'visit motivation': 'Routine follow-up examin...</td>\n",
       "      <td>Doctor: Good morning, Mr. Johnson. How are you...</td>\n",
       "      <td>0.400595</td>\n",
       "      <td>0.964266</td>\n",
       "      <td>8.720000</td>\n",
       "      <td>0.871995</td>\n",
       "      <td>0.739095</td>\n",
       "      <td>0.856995</td>\n",
       "      <td>0.856995</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>39</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>73</td>\n",
       "      <td>0.769641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>530</th>\n",
       "      <td>62901</td>\n",
       "      <td>{'visit motivation': 'Incidentally detected mi...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hi, I understand that you were referre...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>587</th>\n",
       "      <td>2397</td>\n",
       "      <td>{'visit motivation': 'Pain on the lower right ...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, what brings you here today?\\nPa...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>61</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>154957</td>\n",
       "      <td>{'visit motivation': 'Jaundice and dark urine'...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>103</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>103</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>132207</td>\n",
       "      <td>{'visit motivation': 'Pain and swelling on rig...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Good morning, what brings you to the d...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>79</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>79</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>52461</td>\n",
       "      <td>{'visit motivation': 'hypersexuality after sta...</td>\n",
       "      <td>{}</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>58</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "506  171553  {'visit motivation': 'Episodes of continuous a...   \n",
       "494  187015  {'visit motivation': 'Elective left and right ...   \n",
       "444  199095  {'visit motivation': 'Complaining of back pain...   \n",
       "676  183398  {'visit motivation': 'Complaining of atraumati...   \n",
       "97   193172  {'visit motivation': 'Routine follow-up examin...   \n",
       "..      ...                                                ...   \n",
       "530   62901  {'visit motivation': 'Incidentally detected mi...   \n",
       "587    2397  {'visit motivation': 'Pain on the lower right ...   \n",
       "290  154957  {'visit motivation': 'Jaundice and dark urine'...   \n",
       "133  132207  {'visit motivation': 'Pain and swelling on rig...   \n",
       "805   52461  {'visit motivation': 'hypersexuality after sta...   \n",
       "\n",
       "                                                  pred  \\\n",
       "506  {'visit motivation': 'Episodes of continuous a...   \n",
       "494  {'visit motivation': 'Elective left and right ...   \n",
       "444  {'visit motivation': 'Complaining of back pain...   \n",
       "676  {'visit motivation': 'Complaining of atraumati...   \n",
       "97   {'visit motivation': 'Routine follow-up examin...   \n",
       "..                                                 ...   \n",
       "530                                                 {}   \n",
       "587                                                 {}   \n",
       "290                                                 {}   \n",
       "133                                                 {}   \n",
       "805                                                 {}   \n",
       "\n",
       "                                          conversation  mean_bleu  mean_bert  \\\n",
       "506  Doctor: Good morning, Mr. Smith. I see that yo...   0.418091   0.980467   \n",
       "494  Doctor: Good morning, Mr. Johnson. How are you...   0.463583   0.969643   \n",
       "444  Doctor: Hello, how are you feeling today?\\nPat...   0.409682   0.979850   \n",
       "676  Doctor: Hi there, how are you feeling today?\\n...   0.264289   0.982445   \n",
       "97   Doctor: Good morning, Mr. Johnson. How are you...   0.400595   0.964266   \n",
       "..                                                 ...        ...        ...   \n",
       "530  Doctor: Hi, I understand that you were referre...        NaN        NaN   \n",
       "587  Doctor: Hello, what brings you here today?\\nPa...        NaN        NaN   \n",
       "290  Doctor: Hello, how are you feeling today?\\nPat...        NaN        NaN   \n",
       "133  Doctor: Good morning, what brings you to the d...        NaN        NaN   \n",
       "805  Doctor: Hello, how are you feeling today?\\nPat...        NaN        NaN   \n",
       "\n",
       "     mean_gpt_similarity_score  mean_rouge1  mean_rouge2  mean_rougeL  \\\n",
       "506                   9.333333     0.906004     0.767392     0.903359   \n",
       "494                   9.520000     0.899577     0.700834     0.899577   \n",
       "444                   9.040000     0.908114     0.720753     0.899486   \n",
       "676                   9.291667     0.931441     0.735302     0.925330   \n",
       "97                    8.720000     0.871995     0.739095     0.856995   \n",
       "..                         ...          ...          ...          ...   \n",
       "530                        NaN          NaN          NaN          NaN   \n",
       "587                        NaN          NaN          NaN          NaN   \n",
       "290                        NaN          NaN          NaN          NaN   \n",
       "133                        NaN          NaN          NaN          NaN   \n",
       "805                        NaN          NaN          NaN          NaN   \n",
       "\n",
       "     mean_rougeLsum  missing_keys_count  extra_keys_count  common_none_count  \\\n",
       "506        0.903359                  25                 6                 24   \n",
       "494        0.899577                   9                 0                 37   \n",
       "444        0.899486                  12                 0                 30   \n",
       "676        0.925330                   0                 0                 38   \n",
       "97         0.856995                   6                 0                 39   \n",
       "..              ...                 ...               ...                ...   \n",
       "530             NaN                  61                 0                  0   \n",
       "587             NaN                  61                 0                  0   \n",
       "290             NaN                 103                 0                  0   \n",
       "133             NaN                  79                 0                  0   \n",
       "805             NaN                  58                 0                  0   \n",
       "\n",
       "     gold_none_count  pred_none_count  common_non_none  all_keys_count  \\\n",
       "506                1                0               27              83   \n",
       "494                2                0               25              73   \n",
       "444                2                7               25              76   \n",
       "676                2                0               24              64   \n",
       "97                 1                2               25              73   \n",
       "..               ...              ...              ...             ...   \n",
       "530                0                0                0              61   \n",
       "587                0                0                0              61   \n",
       "290                0                0                0             103   \n",
       "133                0                0                0              79   \n",
       "805                0                0                0              58   \n",
       "\n",
       "     aggregated_score  \n",
       "506          0.805180  \n",
       "494          0.792847  \n",
       "444          0.789941  \n",
       "676          0.787908  \n",
       "97           0.769641  \n",
       "..                ...  \n",
       "530          0.000000  \n",
       "587          0.000000  \n",
       "290          0.000000  \n",
       "133          0.000000  \n",
       "805          0.000000  \n",
       "\n",
       "[1000 rows x 19 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>keys</th>\n",
       "      <th>bleu</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>gold_none_prop</th>\n",
       "      <th>pred_none_prop</th>\n",
       "      <th>missing_keys_prop</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>admission/0/care center details</td>\n",
       "      <td>0.264811</td>\n",
       "      <td>0.920767</td>\n",
       "      <td>7.886364</td>\n",
       "      <td>0.733560</td>\n",
       "      <td>0.643519</td>\n",
       "      <td>0.728607</td>\n",
       "      <td>0.728607</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.497743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>admission/0/date</td>\n",
       "      <td>0.222130</td>\n",
       "      <td>0.915933</td>\n",
       "      <td>7.511111</td>\n",
       "      <td>0.744379</td>\n",
       "      <td>0.559630</td>\n",
       "      <td>0.735311</td>\n",
       "      <td>0.735311</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.097</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.541864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>admission/0/duration</td>\n",
       "      <td>0.121161</td>\n",
       "      <td>0.941716</td>\n",
       "      <td>8.843373</td>\n",
       "      <td>0.808594</td>\n",
       "      <td>0.726863</td>\n",
       "      <td>0.806249</td>\n",
       "      <td>0.806249</td>\n",
       "      <td>0.019</td>\n",
       "      <td>0.054</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.632024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>admission/0/reason</td>\n",
       "      <td>0.255464</td>\n",
       "      <td>0.885055</td>\n",
       "      <td>6.565868</td>\n",
       "      <td>0.553590</td>\n",
       "      <td>0.427894</td>\n",
       "      <td>0.533150</td>\n",
       "      <td>0.533150</td>\n",
       "      <td>0.050</td>\n",
       "      <td>0.067</td>\n",
       "      <td>0.014</td>\n",
       "      <td>0.420972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>admission/1/care center details</td>\n",
       "      <td>0.160790</td>\n",
       "      <td>0.945127</td>\n",
       "      <td>8.600000</td>\n",
       "      <td>0.815455</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.815455</td>\n",
       "      <td>0.815455</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.003</td>\n",
       "      <td>0.009</td>\n",
       "      <td>0.704808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>221</th>\n",
       "      <td>treatments/4/time</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.815932</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.026</td>\n",
       "      <td>0.223650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>222</th>\n",
       "      <td>treatments/5/name</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.933010</td>\n",
       "      <td>5.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.563029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>treatments/5/reason for taking</td>\n",
       "      <td>0.762368</td>\n",
       "      <td>0.993797</td>\n",
       "      <td>9.500000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.807692</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.847117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>treatments/5/related condition</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.950773</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.511188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>225</th>\n",
       "      <td>visit motivation</td>\n",
       "      <td>0.322596</td>\n",
       "      <td>0.911850</td>\n",
       "      <td>7.443051</td>\n",
       "      <td>0.676925</td>\n",
       "      <td>0.539898</td>\n",
       "      <td>0.656649</td>\n",
       "      <td>0.656649</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.015</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.576538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>226 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                keys      bleu      bert  \\\n",
       "0    admission/0/care center details  0.264811  0.920767   \n",
       "1                   admission/0/date  0.222130  0.915933   \n",
       "2               admission/0/duration  0.121161  0.941716   \n",
       "3                 admission/0/reason  0.255464  0.885055   \n",
       "4    admission/1/care center details  0.160790  0.945127   \n",
       "..                               ...       ...       ...   \n",
       "221                treatments/4/time  0.000000  0.815932   \n",
       "222                treatments/5/name  0.000000  0.933010   \n",
       "223   treatments/5/reason for taking  0.762368  0.993797   \n",
       "224   treatments/5/related condition  0.000000  0.950773   \n",
       "225                 visit motivation  0.322596  0.911850   \n",
       "\n",
       "     gpt_similarity_score    rouge1    rouge2    rougeL  rougeLsum  \\\n",
       "0                7.886364  0.733560  0.643519  0.728607   0.728607   \n",
       "1                7.511111  0.744379  0.559630  0.735311   0.735311   \n",
       "2                8.843373  0.808594  0.726863  0.806249   0.806249   \n",
       "3                6.565868  0.553590  0.427894  0.533150   0.533150   \n",
       "4                8.600000  0.815455  0.777778  0.815455   0.815455   \n",
       "..                    ...       ...       ...       ...        ...   \n",
       "221              3.000000  0.272727  0.200000  0.272727   0.272727   \n",
       "222              5.500000  0.750000  0.500000  0.750000   0.750000   \n",
       "223              9.500000  0.900000  0.807692  0.900000   0.900000   \n",
       "224              9.000000  0.857143  0.400000  0.571429   0.571429   \n",
       "225              7.443051  0.676925  0.539898  0.656649   0.656649   \n",
       "\n",
       "     gold_none_prop  pred_none_prop  missing_keys_prop  aggregated_score  \n",
       "0             0.050           0.167              0.014          0.497743  \n",
       "1             0.019           0.097              0.014          0.541864  \n",
       "2             0.019           0.054              0.014          0.632024  \n",
       "3             0.050           0.067              0.014          0.420972  \n",
       "4             0.001           0.003              0.009          0.704808  \n",
       "..              ...             ...                ...               ...  \n",
       "221           0.000           0.000              0.026          0.223650  \n",
       "222           0.000           0.000              0.017          0.563029  \n",
       "223           0.000           0.000              0.017          0.847117  \n",
       "224           0.000           0.000              0.017          0.511188  \n",
       "225           0.006           0.015              0.006          0.576538  \n",
       "\n",
       "[226 rows x 12 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "evaluation_path = 'evaluation'\n",
    "eval_by_sample_path = '_eval_res/summary_eval_by_sample.jsonl'\n",
    "eval_by_key_path = '_eval_res/summary_eval_by_key.jsonl'\n",
    "\n",
    "summarizer7_eval_by_sample = load_file(f\"{evaluation_path}/meditron-7b-summarizer{eval_by_sample_path}\")\n",
    "summarizer7_eval_by_key = load_file(f\"{evaluation_path}/meditron-7b-summarizer{eval_by_key_path}\")\n",
    "summarizer13_eval_by_sample = load_file(f\"{evaluation_path}/meditron-13b-summarizer{eval_by_sample_path}\")\n",
    "summarizer13_eval_by_key = load_file(f\"{evaluation_path}/meditron-13b-summarizer{eval_by_key_path}\")\n",
    "\n",
    "display(summarizer7_eval_by_sample.sort_values(by='aggregated_score', ascending=False))\n",
    "display(summarizer7_eval_by_key)\n",
    "display(summarizer13_eval_by_sample.sort_values(by='aggregated_score', ascending=False))\n",
    "display(summarizer13_eval_by_key)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-7b-direct-trunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'done', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done', 'gpt_clarity_score': 'done', 'gpt_coherence_score': 'done', 'gpt_factuality_score': 'done'}\n"
     ]
    }
   ],
   "source": [
    "meditron_7b_direct_trunc_eval_res = note_evaluation('meditron-7b-direct-trunc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-13b-direct-trunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done', 'gpt_clarity_score': 'done', 'gpt_coherence_score': 'tbd', 'gpt_factuality_score': 'tbd'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 200.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n",
      "GPT-4 clarity scores already computed.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 4288.16it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 103.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 869117, total input cost: 8.69117$\n",
      "Sub_batch 1/2: 85 calls, 496747 total tokens: 4.9674700000000005$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 40 calls, 372370 total tokens: 3.7237$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 5407.99it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 61.80it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1529022, total input cost: 15.29022$\n",
      "Sub_batch 1/4: 48 calls, 492530 total tokens: 4.9253$\n",
      "....................20\n",
      "....................40\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 39 calls, 486929 total tokens: 4.8692899999999995$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 34 calls, 487712 total tokens: 4.87712$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 4 calls, 61851 total tokens: 0.61851$\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "meditron_13b_direct_trunc_eval_res = note_evaluation('meditron-13b-direct-trunc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-7b-generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 206.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1819.79it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 96.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 868923, total input cost: 8.68923$\n",
      "Sub_batch 1/2: 85 calls, 497194 total tokens: 4.97194$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 40 calls, 371729 total tokens: 3.7172899999999998$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 5370.10it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 102.13it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 867298, total input cost: 8.67298$\n",
      "Sub_batch 1/2: 85 calls, 496089 total tokens: 4.96089$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 40 calls, 371209 total tokens: 3.71209$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 6291.25it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 61.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1527203, total input cost: 15.27203$\n",
      "Sub_batch 1/4: 49 calls, 493124 total tokens: 4.931240000000001$\n",
      "....................20\n",
      "....................40\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 39 calls, 490780 total tokens: 4.9078$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 34 calls, 496890 total tokens: 4.9689$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 3 calls, 46409 total tokens: 0.46409$\n",
      "...\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "meditron_7b_generator_eval_res = note_evaluation('meditron-7b-generator')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-13b-generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'done', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done', 'gpt_clarity_score': 'done', 'gpt_coherence_score': 'done', 'gpt_factuality_score': 'done'}\n"
     ]
    }
   ],
   "source": [
    "meditron_13b_generator_eval_res = note_evaluation('meditron-13b-generator')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation gpt3-direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:05<00:00, 173.78it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 3711.41it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 111.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 778458, total input cost: 7.78458$\n",
      "Sub_batch 1/2: 93 calls, 497959 total tokens: 4.97959$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 32 calls, 280499 total tokens: 2.80499$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 600.39it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 77.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 776833, total input cost: 7.76833$\n",
      "Sub_batch 1/2: 93 calls, 496750 total tokens: 4.9675$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 32 calls, 280083 total tokens: 2.8008300000000004$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 4787.41it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 65.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1436738, total input cost: 14.36738$\n",
      "Sub_batch 1/3: 52 calls, 490436 total tokens: 4.90436$\n",
      "....................20\n",
      "....................40\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 41 calls, 493053 total tokens: 4.93053$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 32 calls, 453249 total tokens: 4.53249$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "gpt3_direct_eval_res = note_evaluation('gpt3-direct')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation mistral-7b-direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:06<00:00, 158.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 5926.70it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 95.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 778041, total input cost: 7.780410000000001$\n",
      "Sub_batch 1/2: 93 calls, 495414 total tokens: 4.95414$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 32 calls, 282627 total tokens: 2.82627$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 11104.97it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:00<00:00, 127.46it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 776416, total input cost: 7.76416$\n",
      "Sub_batch 1/2: 93 calls, 494205 total tokens: 4.94205$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 32 calls, 282211 total tokens: 2.8221100000000003$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 6524.00it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 70.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1436321, total input cost: 14.363209999999999$\n",
      "Sub_batch 1/3: 52 calls, 488405 total tokens: 4.88405$\n",
      "....................20\n",
      "....................40\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 41 calls, 496115 total tokens: 4.96115$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 32 calls, 451801 total tokens: 4.51801$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "mistral_7_b_eval_res = note_evaluation('mistral-7b-direct')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation gpt3-generator-gpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done', 'gpt_clarity_score': 'in progress', 'gpt_coherence_score': 'tbd', 'gpt_factuality_score': 'tbd'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:06<00:00, 150.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n",
      "GPT-4 clarity score computation already in progress, resuming\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 40/40 [00:00<00:00, 10840.10it/s]\n",
      "Building sub-batches: 100%|██████████| 40/40 [00:00<00:00, 94.97it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 182538, total input cost: 1.8253800000000002$\n",
      "Sub_batch 1/1: 40 calls, 182538 total tokens: 1.8253800000000002$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 11788.11it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 107.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 861110, total input cost: 8.6111$\n",
      "Sub_batch 1/2: 86 calls, 499534 total tokens: 4.99534$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 39 calls, 361576 total tokens: 3.6157600000000003$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 2478.63it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 51.75it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1521015, total input cost: 15.21015$\n",
      "Sub_batch 1/4: 49 calls, 498818 total tokens: 4.98818$\n",
      "....................20\n",
      "....................40\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 40 calls, 496067 total tokens: 4.96067$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 34 calls, 495254 total tokens: 4.95254$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 2 calls, 30876 total tokens: 0.30876000000000003$\n",
      "..\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "gpt3_generator_gpt_eval_res = note_evaluation('gpt3-generator-gpt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation gpt3-generator-7b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[autoreload of utils.eval failed: Traceback (most recent call last):\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
      "    superreload(m, reload, self.old_objects)\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
      "    module = reload(module)\n",
      "             ^^^^^^^^^^^^^^\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/importlib/__init__.py\", line 169, in reload\n",
      "    _bootstrap._exec(spec, module)\n",
      "  File \"<frozen importlib._bootstrap>\", line 621, in _exec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 940, in exec_module\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"/Users/paul/Desktop/Make-Project/ClinicalNotes/utils/eval.py\", line 64, in <module>\n",
      "    ELO_MODELS = MODELS_TO_MODE.keys().copy()\n",
      "                 ^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "AttributeError: 'dict_keys' object has no attribute 'copy'\n",
      "]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 211.05it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 6572.41it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 84.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 857331, total input cost: 8.573310000000001$\n",
      "Sub_batch 1/2: 86 calls, 496962 total tokens: 4.96962$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 39 calls, 360369 total tokens: 3.6036900000000003$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 13186.98it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 109.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 855706, total input cost: 8.55706$\n",
      "Sub_batch 1/2: 86 calls, 495844 total tokens: 4.95844$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 39 calls, 359862 total tokens: 3.5986200000000004$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 7522.39it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 64.28it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1515611, total input cost: 15.156110000000002$\n",
      "Sub_batch 1/4: 49 calls, 490702 total tokens: 4.90702$\n",
      "....................20\n",
      "....................40\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 39 calls, 489192 total tokens: 4.89192$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 34 calls, 488411 total tokens: 4.88411$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 3 calls, 47306 total tokens: 0.47306$\n",
      "...\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "gpt3_generator_7b = note_evaluation('gpt3-generator-7b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation gpt3-generator-13b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 230.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 7118.84it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 109.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 856862, total input cost: 8.56862$\n",
      "Sub_batch 1/2: 86 calls, 497552 total tokens: 4.97552$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 39 calls, 359310 total tokens: 3.5931$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 10036.91it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 99.48it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 855237, total input cost: 8.55237$\n",
      "Sub_batch 1/2: 86 calls, 496434 total tokens: 4.96434$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 39 calls, 358803 total tokens: 3.58803$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 5663.33it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 52.16it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1515142, total input cost: 15.151420000000002$\n",
      "Sub_batch 1/4: 49 calls, 486296 total tokens: 4.86296$\n",
      "....................20\n",
      "....................40\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 40 calls, 499574 total tokens: 4.9957400000000005$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 34 calls, 499232 total tokens: 4.99232$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 2 calls, 30040 total tokens: 0.3004$\n",
      "..\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "gpt3_generator_13b_eval_res = note_evaluation('gpt3-generator-13b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation  meditron-7b-generator-gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 226.03it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 7316.74it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 102.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 876173, total input cost: 8.76173$\n",
      "Sub_batch 1/2: 84 calls, 494207 total tokens: 4.94207$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 41 calls, 381966 total tokens: 3.8196600000000003$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 6162.44it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 65.73it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 874548, total input cost: 8.74548$\n",
      "Sub_batch 1/2: 84 calls, 493115 total tokens: 4.931150000000001$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "....................80\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/2: 41 calls, 381433 total tokens: 3.81433$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 5024.90it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 65.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1534453, total input cost: 15.34453$\n",
      "Sub_batch 1/4: 49 calls, 493320 total tokens: 4.9332$\n",
      "....................20\n",
      "....................40\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 40 calls, 497951 total tokens: 4.97951$\n",
      "....................20\n",
      "....................40\n",
      "\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 33 calls, 494577 total tokens: 4.94577$\n",
      "....................20\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 3 calls, 48605 total tokens: 0.48605$\n",
      "...\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "meditron_7b_generator_gold_eval_res = note_evaluation('meditron-7b-generator-gold')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation meditron-13b-generator-gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'done'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:04<00:00, 232.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n",
      "GPT-4 scores already computed.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 3361.29it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 95.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 871662, total input cost: 8.71662$\n",
      "Sub_batch 1/3: 72 calls, 396093 total tokens: 3.9609300000000003$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 45 calls, 393591 total tokens: 3.9359100000000002$\n",
      "....................20\n",
      "....................40\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 8 calls, 81978 total tokens: 0.81978$\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 8730.71it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 91.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 870037, total input cost: 8.700370000000001$\n",
      "Sub_batch 1/3: 72 calls, 395157 total tokens: 3.95157$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 45 calls, 393006 total tokens: 3.9300599999999997$\n",
      "....................20\n",
      "....................40\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 8 calls, 81874 total tokens: 0.81874$\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 4720.34it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:03<00:00, 39.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1529942, total input cost: 15.29942$\n",
      "Sub_batch 1/4: 39 calls, 393587 total tokens: 3.93587$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 34 calls, 396787 total tokens: 3.96787$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 29 calls, 390117 total tokens: 3.9011700000000005$\n",
      "....................20\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 23 calls, 349451 total tokens: 3.4945100000000004$\n",
      "....................20\n",
      "...\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "meditron_13b_generator_gold_eval_res = note_evaluation('meditron-13b-generator-gold')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation gpt4-direct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'scores': 'in progress', 'bleu': 'done', 'rouge': 'done', 'bert': 'done', 'gpt_similarity_score': 'tbd', 'gpt_clarity_score': 'tbd', 'gpt_coherence_score': 'tbd', 'gpt_factuality_score': 'tbd'}\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:05<00:00, 198.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n",
      "BLEU scores already computed.\n",
      "ROUGE scores already computed.\n",
      "Computing BERTscores...\n",
      "BERTscores already computed.\n",
      "GPT-4 similarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1626.71it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 75.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 915430, total input cost: 9.1543$\n",
      "Sub_batch 1/3: 54 calls, 394930 total tokens: 3.9493$\n",
      "....................20\n",
      "....................40\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 54 calls, 396375 total tokens: 3.96375$\n",
      "....................20\n",
      "....................40\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 17 calls, 124125 total tokens: 1.24125$\n",
      ".................\n",
      "\n",
      "Break for 7 seconds.\n",
      "End of break.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 7812.48it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 88.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 912656, total input cost: 9.12656$\n",
      "Sub_batch 1/3: 69 calls, 397673 total tokens: 3.9767300000000003$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 44 calls, 390190 total tokens: 3.9019$\n",
      "....................20\n",
      "....................40\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 12 calls, 124793 total tokens: 1.24793$\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 8308.45it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 76.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 911031, total input cost: 9.11031$\n",
      "Sub_batch 1/3: 69 calls, 396776 total tokens: 3.96776$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 45 calls, 399950 total tokens: 3.9995$\n",
      "....................20\n",
      "....................40\n",
      ".....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 11 calls, 114305 total tokens: 1.1430500000000001$\n",
      "...........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 6754.20it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 50.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1570936, total input cost: 15.70936$\n",
      "Sub_batch 1/4: 39 calls, 397691 total tokens: 3.9769099999999997$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 33 calls, 399373 total tokens: 3.99373$\n",
      "....................20\n",
      ".............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 29 calls, 398897 total tokens: 3.98897$\n",
      "....................20\n",
      ".........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 24 calls, 374975 total tokens: 3.74975$\n",
      "....................20\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "gpt4_direct_eval_res = note_evaluation('gpt4-direct')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation llama-2-7b-chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Evaluation directory and progress monitoring\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:05<00:00, 196.22it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing BLEU scores: 100%|██████████| 1000/1000 [00:06<00:00, 150.67it/s]\n",
      "Computing Rouge scores: 100%|██████████| 10/10 [03:02<00:00, 18.26s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing BERTscores...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing BERT scores: 100%|██████████| 10/10 [26:13<00:00, 157.33s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-4 similarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 544.75it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:04<00:00, 27.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 946116, total input cost: 9.46116$\n",
      "Sub_batch 1/3: 52 calls, 394029 total tokens: 3.94029$\n",
      "....................20\n",
      "....................40\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 52 calls, 395181 total tokens: 3.95181$\n",
      "....................20\n",
      "....................40\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 21 calls, 156906 total tokens: 1.5690600000000001$\n",
      "....................20\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 7791.01it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 72.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 943334, total input cost: 9.43334$\n",
      "Sub_batch 1/3: 67 calls, 397648 total tokens: 3.9764800000000005$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      ".......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 44 calls, 398733 total tokens: 3.98733$\n",
      "....................20\n",
      "....................40\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 14 calls, 146953 total tokens: 1.46953$\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 680.75it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 58.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 941709, total input cost: 9.41709$\n",
      "Sub_batch 1/3: 67 calls, 396777 total tokens: 3.96777$\n",
      "....................20\n",
      "....................40\n",
      "....................60\n",
      ".......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 44 calls, 398161 total tokens: 3.9816100000000003$\n",
      "....................20\n",
      "....................40\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 14 calls, 146771 total tokens: 1.4677099999999998$\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1948.32it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:08<00:00, 13.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1570997, total input cost: 15.70997$\n",
      "Sub_batch 1/4: 39 calls, 399183 total tokens: 3.99183$\n",
      "....................20\n",
      "...................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/4: 32 calls, 386883 total tokens: 3.86883$\n",
      "....................20\n",
      "............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/4: 28 calls, 388136 total tokens: 3.8813600000000004$\n",
      "....................20\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/4: 26 calls, 396795 total tokens: 3.96795$\n",
      "....................20\n",
      "......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "llama_2_7b_chat_eval_res = note_evaluation('llama-2-7b-direct')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation llama-2-13b-chat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[autoreload of utils.eval failed: Traceback (most recent call last):\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 276, in check\n",
      "    superreload(m, reload, self.old_objects)\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/site-packages/IPython/extensions/autoreload.py\", line 475, in superreload\n",
      "    module = reload(module)\n",
      "             ^^^^^^^^^^^^^^\n",
      "  File \"/Users/paul/anaconda3/envs/ClinicalNotes/lib/python3.11/importlib/__init__.py\", line 169, in reload\n",
      "    _bootstrap._exec(spec, module)\n",
      "  File \"<frozen importlib._bootstrap>\", line 621, in _exec\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 936, in exec_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1074, in get_code\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 1004, in source_to_code\n",
      "  File \"<frozen importlib._bootstrap>\", line 241, in _call_with_frames_removed\n",
      "  File \"/Users/paul/Desktop/Make-Project/ClinicalNotes/utils/eval.py\", line 121\n",
      "    outputs_notes['notes'] = outputs_notes.apply(lambda row: [(row[modelA], row[modelB]) for (modelA,modelB) in row['random_comparisons'] , axis=1)\n",
      "                                                                                                                                                  ^\n",
      "SyntaxError: closing parenthesis ')' does not match opening parenthesis '['\n",
      "]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Evaluation directory and progress monitoring\n",
      "Removing stopwords...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:10<00:00, 99.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stopwords removed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing BLEU scores: 100%|██████████| 1000/1000 [00:12<00:00, 77.04it/s]\n",
      "Computing Rouge scores: 100%|██████████| 10/10 [04:49<00:00, 28.92s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing BERTscores...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing BERT scores: 100%|██████████| 10/10 [20:48<00:00, 124.87s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPT-4 similarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1293.59it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 61.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1120307, total input cost: 11.20307$\n",
      "Sub_batch 1/3: 44 calls, 394739 total tokens: 3.94739$\n",
      "....................20\n",
      "....................40\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 44 calls, 392317 total tokens: 3.9231700000000003$\n",
      "....................20\n",
      "....................40\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 37 calls, 333251 total tokens: 3.3325099999999996$\n",
      "....................20\n",
      ".................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT clarity scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1133.76it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 42.04it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1117523, total input cost: 11.175229999999999$\n",
      "Sub_batch 1/3: 56 calls, 394348 total tokens: 3.94348$\n",
      "....................20\n",
      "....................40\n",
      "................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 41 calls, 395953 total tokens: 3.95953$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 28 calls, 327222 total tokens: 3.27222$\n",
      "....................20\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT coherence scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 1256.58it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:01<00:00, 70.98it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1115898, total input cost: 11.15898$\n",
      "Sub_batch 1/3: 56 calls, 393620 total tokens: 3.9362$\n",
      "....................20\n",
      "....................40\n",
      "................\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/3: 41 calls, 395420 total tokens: 3.9542$\n",
      "....................20\n",
      "....................40\n",
      ".\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/3: 28 calls, 326858 total tokens: 3.26858$\n",
      "....................20\n",
      "........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "GPT factuality scoring...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building score prompts: 100%|██████████| 125/125 [00:00<00:00, 2449.06it/s]\n",
      "Building sub-batches: 100%|██████████| 125/125 [00:02<00:00, 42.66it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total input tokens: 1745186, total input cost: 17.45186$\n",
      "Sub_batch 1/5: 34 calls, 390748 total tokens: 3.90748$\n",
      "....................20\n",
      "..............\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 2/5: 30 calls, 397021 total tokens: 3.9702100000000002$\n",
      "....................20\n",
      "..........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 3/5: 27 calls, 396947 total tokens: 3.9694700000000003$\n",
      "....................20\n",
      ".......\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 4/5: 24 calls, 385271 total tokens: 3.85271$\n",
      "....................20\n",
      "....\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n",
      "Sub_batch 5/5: 10 calls, 175199 total tokens: 1.7519900000000002$\n",
      "..........\n",
      "\n",
      "Break for 5 seconds.\n",
      "End of break.\n"
     ]
    }
   ],
   "source": [
    "llama_2_13b_chat_eval_res = note_evaluation('llama-2-13b-direct')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merging note evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_eval = merge_note_evaluations(model_names= NOTE_MODELS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>bleu</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>0.439479</td>\n",
       "      <td>0.899691</td>\n",
       "      <td>6.364</td>\n",
       "      <td>7.089</td>\n",
       "      <td>7.390</td>\n",
       "      <td>9.177</td>\n",
       "      <td>0.729841</td>\n",
       "      <td>0.563817</td>\n",
       "      <td>0.665678</td>\n",
       "      <td>0.665678</td>\n",
       "      <td>0.645634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>0.434462</td>\n",
       "      <td>0.898802</td>\n",
       "      <td>6.533</td>\n",
       "      <td>7.145</td>\n",
       "      <td>7.409</td>\n",
       "      <td>9.282</td>\n",
       "      <td>0.729459</td>\n",
       "      <td>0.563043</td>\n",
       "      <td>0.664638</td>\n",
       "      <td>0.664638</td>\n",
       "      <td>0.645752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>0.251536</td>\n",
       "      <td>0.859992</td>\n",
       "      <td>5.360</td>\n",
       "      <td>7.045</td>\n",
       "      <td>7.219</td>\n",
       "      <td>7.998</td>\n",
       "      <td>0.535441</td>\n",
       "      <td>0.331078</td>\n",
       "      <td>0.386767</td>\n",
       "      <td>0.386767</td>\n",
       "      <td>0.429350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>0.246159</td>\n",
       "      <td>0.860041</td>\n",
       "      <td>5.345</td>\n",
       "      <td>7.126</td>\n",
       "      <td>7.398</td>\n",
       "      <td>8.119</td>\n",
       "      <td>0.537501</td>\n",
       "      <td>0.329094</td>\n",
       "      <td>0.388867</td>\n",
       "      <td>0.388867</td>\n",
       "      <td>0.430792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>0.200022</td>\n",
       "      <td>0.847101</td>\n",
       "      <td>6.739</td>\n",
       "      <td>6.944</td>\n",
       "      <td>7.055</td>\n",
       "      <td>8.539</td>\n",
       "      <td>0.479645</td>\n",
       "      <td>0.293181</td>\n",
       "      <td>0.352149</td>\n",
       "      <td>0.352149</td>\n",
       "      <td>0.399896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>0.138026</td>\n",
       "      <td>0.833835</td>\n",
       "      <td>4.132</td>\n",
       "      <td>6.005</td>\n",
       "      <td>6.094</td>\n",
       "      <td>8.025</td>\n",
       "      <td>0.387875</td>\n",
       "      <td>0.224890</td>\n",
       "      <td>0.271220</td>\n",
       "      <td>0.271220</td>\n",
       "      <td>0.315527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>0.223627</td>\n",
       "      <td>0.852028</td>\n",
       "      <td>6.274</td>\n",
       "      <td>7.005</td>\n",
       "      <td>7.134</td>\n",
       "      <td>8.326</td>\n",
       "      <td>0.549584</td>\n",
       "      <td>0.364188</td>\n",
       "      <td>0.441554</td>\n",
       "      <td>0.441554</td>\n",
       "      <td>0.462089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>0.235334</td>\n",
       "      <td>0.872546</td>\n",
       "      <td>8.295</td>\n",
       "      <td>7.962</td>\n",
       "      <td>8.291</td>\n",
       "      <td>9.481</td>\n",
       "      <td>0.588361</td>\n",
       "      <td>0.377739</td>\n",
       "      <td>0.498093</td>\n",
       "      <td>0.498093</td>\n",
       "      <td>0.512879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>0.187369</td>\n",
       "      <td>0.849517</td>\n",
       "      <td>7.909</td>\n",
       "      <td>7.854</td>\n",
       "      <td>8.051</td>\n",
       "      <td>8.895</td>\n",
       "      <td>0.520189</td>\n",
       "      <td>0.302732</td>\n",
       "      <td>0.401866</td>\n",
       "      <td>0.401866</td>\n",
       "      <td>0.438813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>0.249655</td>\n",
       "      <td>0.869048</td>\n",
       "      <td>8.394</td>\n",
       "      <td>7.639</td>\n",
       "      <td>7.793</td>\n",
       "      <td>8.853</td>\n",
       "      <td>0.576516</td>\n",
       "      <td>0.377367</td>\n",
       "      <td>0.388395</td>\n",
       "      <td>0.388395</td>\n",
       "      <td>0.464545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>0.248659</td>\n",
       "      <td>0.869052</td>\n",
       "      <td>8.308</td>\n",
       "      <td>7.682</td>\n",
       "      <td>7.769</td>\n",
       "      <td>8.860</td>\n",
       "      <td>0.575296</td>\n",
       "      <td>0.376052</td>\n",
       "      <td>0.389251</td>\n",
       "      <td>0.389251</td>\n",
       "      <td>0.464089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>0.250178</td>\n",
       "      <td>0.869380</td>\n",
       "      <td>8.226</td>\n",
       "      <td>7.625</td>\n",
       "      <td>7.823</td>\n",
       "      <td>8.800</td>\n",
       "      <td>0.578522</td>\n",
       "      <td>0.378426</td>\n",
       "      <td>0.390204</td>\n",
       "      <td>0.390204</td>\n",
       "      <td>0.465134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>0.360376</td>\n",
       "      <td>0.884114</td>\n",
       "      <td>7.424</td>\n",
       "      <td>7.061</td>\n",
       "      <td>7.308</td>\n",
       "      <td>8.890</td>\n",
       "      <td>0.639896</td>\n",
       "      <td>0.452532</td>\n",
       "      <td>0.485576</td>\n",
       "      <td>0.485576</td>\n",
       "      <td>0.531658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>0.368051</td>\n",
       "      <td>0.887227</td>\n",
       "      <td>7.428</td>\n",
       "      <td>7.214</td>\n",
       "      <td>7.453</td>\n",
       "      <td>8.994</td>\n",
       "      <td>0.656079</td>\n",
       "      <td>0.464628</td>\n",
       "      <td>0.499375</td>\n",
       "      <td>0.499375</td>\n",
       "      <td>0.544340</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          model      bleu      bert  gpt_similarity_score  \\\n",
       "0      meditron-7b-direct-trunc  0.439479  0.899691                 6.364   \n",
       "1     meditron-13b-direct-trunc  0.434462  0.898802                 6.533   \n",
       "2         meditron-7b-generator  0.251536  0.859992                 5.360   \n",
       "3        meditron-13b-generator  0.246159  0.860041                 5.345   \n",
       "4             llama-2-7b-direct  0.200022  0.847101                 6.739   \n",
       "5            llama-2-13b-direct  0.138026  0.833835                 4.132   \n",
       "6             mistral-7b-direct  0.223627  0.852028                 6.274   \n",
       "7                   gpt3-direct  0.235334  0.872546                 8.295   \n",
       "8                   gpt4-direct  0.187369  0.849517                 7.909   \n",
       "9            gpt3-generator-gpt  0.249655  0.869048                 8.394   \n",
       "10            gpt3-generator-7b  0.248659  0.869052                 8.308   \n",
       "11           gpt3-generator-13b  0.250178  0.869380                 8.226   \n",
       "12   meditron-7b-generator-gold  0.360376  0.884114                 7.424   \n",
       "13  meditron-13b-generator-gold  0.368051  0.887227                 7.428   \n",
       "\n",
       "    gpt_clarity_score  gpt_coherence_score  gpt_factuality_score    rouge1  \\\n",
       "0               7.089                7.390                 9.177  0.729841   \n",
       "1               7.145                7.409                 9.282  0.729459   \n",
       "2               7.045                7.219                 7.998  0.535441   \n",
       "3               7.126                7.398                 8.119  0.537501   \n",
       "4               6.944                7.055                 8.539  0.479645   \n",
       "5               6.005                6.094                 8.025  0.387875   \n",
       "6               7.005                7.134                 8.326  0.549584   \n",
       "7               7.962                8.291                 9.481  0.588361   \n",
       "8               7.854                8.051                 8.895  0.520189   \n",
       "9               7.639                7.793                 8.853  0.576516   \n",
       "10              7.682                7.769                 8.860  0.575296   \n",
       "11              7.625                7.823                 8.800  0.578522   \n",
       "12              7.061                7.308                 8.890  0.639896   \n",
       "13              7.214                7.453                 8.994  0.656079   \n",
       "\n",
       "      rouge2    rougeL  rougeLsum  aggregated_score  \n",
       "0   0.563817  0.665678   0.665678          0.645634  \n",
       "1   0.563043  0.664638   0.664638          0.645752  \n",
       "2   0.331078  0.386767   0.386767          0.429350  \n",
       "3   0.329094  0.388867   0.388867          0.430792  \n",
       "4   0.293181  0.352149   0.352149          0.399896  \n",
       "5   0.224890  0.271220   0.271220          0.315527  \n",
       "6   0.364188  0.441554   0.441554          0.462089  \n",
       "7   0.377739  0.498093   0.498093          0.512879  \n",
       "8   0.302732  0.401866   0.401866          0.438813  \n",
       "9   0.377367  0.388395   0.388395          0.464545  \n",
       "10  0.376052  0.389251   0.389251          0.464089  \n",
       "11  0.378426  0.390204   0.390204          0.465134  \n",
       "12  0.452532  0.485576   0.485576          0.531658  \n",
       "13  0.464628  0.499375   0.499375          0.544340  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(final_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-7b-direct-trunc\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>840</th>\n",
       "      <td>79021</td>\n",
       "      <td>A 65-year-old woman was referred to our interv...</td>\n",
       "      <td>A 62-year-old woman was referred to our interv...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 65-year-old woman was referred to ...</td>\n",
       "      <td>840</td>\n",
       "      <td>0.919281</td>\n",
       "      <td>0.992771</td>\n",
       "      <td>0.983051</td>\n",
       "      <td>0.992771</td>\n",
       "      <td>0.992771</td>\n",
       "      <td>0.993514</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.942579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>904</th>\n",
       "      <td>18114</td>\n",
       "      <td>At the Vascular Surgery Department of our Inst...</td>\n",
       "      <td>We present the case of a 32-year-old Caucasian...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'At the Vascular Surgery Department o...</td>\n",
       "      <td>904</td>\n",
       "      <td>0.451109</td>\n",
       "      <td>0.987654</td>\n",
       "      <td>0.977667</td>\n",
       "      <td>0.987654</td>\n",
       "      <td>0.987654</td>\n",
       "      <td>0.996472</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.930214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>624</th>\n",
       "      <td>35770</td>\n",
       "      <td>A 37-year-old healthy woman complained of recu...</td>\n",
       "      <td>A 24-year-old Caucasian woman presented to our...</td>\n",
       "      <td>Doctor: Hello, how can I help you today?\\nPati...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 37-year-old healthy woman complain...</td>\n",
       "      <td>624</td>\n",
       "      <td>0.357638</td>\n",
       "      <td>0.995327</td>\n",
       "      <td>0.985915</td>\n",
       "      <td>0.995327</td>\n",
       "      <td>0.995327</td>\n",
       "      <td>0.995337</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.921549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>251</th>\n",
       "      <td>103646</td>\n",
       "      <td>A 25-year-old male, without any comorbidities,...</td>\n",
       "      <td>A 33-year-old male, involved in a road traffic...</td>\n",
       "      <td>Doctor: Hello, what brings you to the emergenc...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 25-year-old male, without any como...</td>\n",
       "      <td>251</td>\n",
       "      <td>0.715163</td>\n",
       "      <td>0.984456</td>\n",
       "      <td>0.973958</td>\n",
       "      <td>0.984456</td>\n",
       "      <td>0.984456</td>\n",
       "      <td>0.981494</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>0.921101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>610</th>\n",
       "      <td>117254</td>\n",
       "      <td>A 21-year-old young man, who had progressive p...</td>\n",
       "      <td>A 43-year-old woman was admitted to our hospit...</td>\n",
       "      <td>Doctor: Hello, what brings you to the hospital...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 21-year-old young man, who had pro...</td>\n",
       "      <td>610</td>\n",
       "      <td>0.866014</td>\n",
       "      <td>0.958974</td>\n",
       "      <td>0.922680</td>\n",
       "      <td>0.958974</td>\n",
       "      <td>0.958974</td>\n",
       "      <td>0.981152</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0.917818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812</th>\n",
       "      <td>80995</td>\n",
       "      <td>Patient 1. The proband is a 25-year-old Egypti...</td>\n",
       "      <td>A 23-year-old male was referred to the genetic...</td>\n",
       "      <td>Doctor: Good morning, Mr. Patient. I understan...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'Patient 1. The proband is a 25-year-...</td>\n",
       "      <td>812</td>\n",
       "      <td>0.194425</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.108718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>39899</td>\n",
       "      <td>We present the case of a Caucasian male infant...</td>\n",
       "      <td>A male infant was born to a 17-year-old Africa...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'We present the case of a Caucasian m...</td>\n",
       "      <td>287</td>\n",
       "      <td>0.347458</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.098408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>653</th>\n",
       "      <td>136715</td>\n",
       "      <td>A 22-year-old female, weighing 53 kg, with a h...</td>\n",
       "      <td>A 28-year-old female, diagnosed case of SLE on...</td>\n",
       "      <td>Doctor: Good morning, what brings you to the h...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 22-year-old female, weighing 53 kg...</td>\n",
       "      <td>653</td>\n",
       "      <td>0.327465</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.096827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>25927</td>\n",
       "      <td>An 82-year-old female initially presented with...</td>\n",
       "      <td>A 56-year-old female, with an unremarkable pas...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'An 82-year-old female initially pres...</td>\n",
       "      <td>59</td>\n",
       "      <td>0.264567</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.091990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>36889</td>\n",
       "      <td>A 74-year-old female patient was admitted to o...</td>\n",
       "      <td>A 72-year-old woman was admitted to our Cardio...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 74-year-old female patient was adm...</td>\n",
       "      <td>572</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.089048</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "840   79021  A 65-year-old woman was referred to our interv...   \n",
       "904   18114  At the Vascular Surgery Department of our Inst...   \n",
       "624   35770  A 37-year-old healthy woman complained of recu...   \n",
       "251  103646  A 25-year-old male, without any comorbidities,...   \n",
       "610  117254  A 21-year-old young man, who had progressive p...   \n",
       "..      ...                                                ...   \n",
       "812   80995  Patient 1. The proband is a 25-year-old Egypti...   \n",
       "287   39899  We present the case of a Caucasian male infant...   \n",
       "653  136715  A 22-year-old female, weighing 53 kg, with a h...   \n",
       "59    25927  An 82-year-old female initially presented with...   \n",
       "572   36889  A 74-year-old female patient was admitted to o...   \n",
       "\n",
       "                                                  pred  \\\n",
       "840  A 62-year-old woman was referred to our interv...   \n",
       "904  We present the case of a 32-year-old Caucasian...   \n",
       "624  A 24-year-old Caucasian woman presented to our...   \n",
       "251  A 33-year-old male, involved in a road traffic...   \n",
       "610  A 43-year-old woman was admitted to our hospit...   \n",
       "..                                                 ...   \n",
       "812  A 23-year-old male was referred to the genetic...   \n",
       "287  A male infant was born to a 17-year-old Africa...   \n",
       "653  A 28-year-old female, diagnosed case of SLE on...   \n",
       "59   A 56-year-old female, with an unremarkable pas...   \n",
       "572  A 72-year-old woman was admitted to our Cardio...   \n",
       "\n",
       "                                          conversation  \\\n",
       "840  Doctor: Hello, how are you feeling today?\\nPat...   \n",
       "904  Doctor: Hello, how are you feeling today?\\nPat...   \n",
       "624  Doctor: Hello, how can I help you today?\\nPati...   \n",
       "251  Doctor: Hello, what brings you to the emergenc...   \n",
       "610  Doctor: Hello, what brings you to the hospital...   \n",
       "..                                                 ...   \n",
       "812  Doctor: Good morning, Mr. Patient. I understan...   \n",
       "287  Doctor: Good morning, how are you today?\\nPati...   \n",
       "653  Doctor: Good morning, what brings you to the h...   \n",
       "59   Doctor: Good morning, how are you feeling toda...   \n",
       "572  Doctor: Good morning, how are you today?\\nPati...   \n",
       "\n",
       "                   model_name  \\\n",
       "840  meditron-7b-direct-trunc   \n",
       "904  meditron-7b-direct-trunc   \n",
       "624  meditron-7b-direct-trunc   \n",
       "251  meditron-7b-direct-trunc   \n",
       "610  meditron-7b-direct-trunc   \n",
       "..                        ...   \n",
       "812  meditron-7b-direct-trunc   \n",
       "287  meditron-7b-direct-trunc   \n",
       "653  meditron-7b-direct-trunc   \n",
       "59   meditron-7b-direct-trunc   \n",
       "572  meditron-7b-direct-trunc   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "840  {'gold': 'A 65-year-old woman was referred to ...   840  0.919281   \n",
       "904  {'gold': 'At the Vascular Surgery Department o...   904  0.451109   \n",
       "624  {'gold': 'A 37-year-old healthy woman complain...   624  0.357638   \n",
       "251  {'gold': 'A 25-year-old male, without any como...   251  0.715163   \n",
       "610  {'gold': 'A 21-year-old young man, who had pro...   610  0.866014   \n",
       "..                                                 ...   ...       ...   \n",
       "812  {'gold': 'Patient 1. The proband is a 25-year-...   812  0.194425   \n",
       "287  {'gold': 'We present the case of a Caucasian m...   287  0.347458   \n",
       "653  {'gold': 'A 22-year-old female, weighing 53 kg...   653  0.327465   \n",
       "59   {'gold': 'An 82-year-old female initially pres...    59  0.264567   \n",
       "572  {'gold': 'A 74-year-old female patient was adm...   572  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "840  0.992771  0.983051  0.992771   0.992771  0.993514                     4   \n",
       "904  0.987654  0.977667  0.987654   0.987654  0.996472                    10   \n",
       "624  0.995327  0.985915  0.995327   0.995327  0.995337                     9   \n",
       "251  0.984456  0.973958  0.984456   0.984456  0.981494                     9   \n",
       "610  0.958974  0.922680  0.958974   0.958974  0.981152                     8   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "812  0.000000  0.000000  0.000000   0.000000  0.000000                     9   \n",
       "287  0.000000  0.000000  0.000000   0.000000  0.000000                     2   \n",
       "653  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "59   0.000000  0.000000  0.000000   0.000000  0.000000                     2   \n",
       "572  0.000000  0.000000  0.000000   0.000000  0.000000                     6   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "840                  7                    7                    10   \n",
       "904                  7                    9                    10   \n",
       "624                  6                    9                    10   \n",
       "251                  4                    6                     9   \n",
       "610                  7                    7                     9   \n",
       "..                 ...                  ...                   ...   \n",
       "812                  6                    7                    10   \n",
       "287                  6                    8                    10   \n",
       "653                  9                    6                    10   \n",
       "59                   8                    6                    10   \n",
       "572                  8                    7                    10   \n",
       "\n",
       "     aggregated_score  \n",
       "840          0.942579  \n",
       "904          0.930214  \n",
       "624          0.921549  \n",
       "251          0.921101  \n",
       "610          0.917818  \n",
       "..                ...  \n",
       "812          0.108718  \n",
       "287          0.098408  \n",
       "653          0.096827  \n",
       "59           0.091990  \n",
       "572          0.089048  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-13b-direct-trunc\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>86402</td>\n",
       "      <td>A 27-year-old female patient with good general...</td>\n",
       "      <td>A 25-year-old female patient in good general h...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 27-year-old female patient with go...</td>\n",
       "      <td>699</td>\n",
       "      <td>0.960636</td>\n",
       "      <td>0.995455</td>\n",
       "      <td>0.990868</td>\n",
       "      <td>0.995455</td>\n",
       "      <td>0.995455</td>\n",
       "      <td>0.996016</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.981765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>832</th>\n",
       "      <td>108907</td>\n",
       "      <td>A 45-year-old male patient presented to us who...</td>\n",
       "      <td>A 45-year-old male patient presented to the cl...</td>\n",
       "      <td>Doctor: Good morning, sir. What brings you her...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 45-year-old male patient presented...</td>\n",
       "      <td>832</td>\n",
       "      <td>0.995927</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999406</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.978919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>115484</td>\n",
       "      <td>A 71-year-old female patient was referred to t...</td>\n",
       "      <td>A 54-year-old female patient with a giant abdo...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 71-year-old female patient was ref...</td>\n",
       "      <td>197</td>\n",
       "      <td>0.987310</td>\n",
       "      <td>0.990909</td>\n",
       "      <td>0.986301</td>\n",
       "      <td>0.990909</td>\n",
       "      <td>0.990909</td>\n",
       "      <td>0.997629</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>0.971239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>139279</td>\n",
       "      <td>A 24-year-old female patient was referred to t...</td>\n",
       "      <td>A 28-year-old female patient was referred to t...</td>\n",
       "      <td>Doctor: Hello, How are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 24-year-old female patient was ref...</td>\n",
       "      <td>342</td>\n",
       "      <td>0.974654</td>\n",
       "      <td>0.992806</td>\n",
       "      <td>0.987952</td>\n",
       "      <td>0.992806</td>\n",
       "      <td>0.992806</td>\n",
       "      <td>0.992954</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.970078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>202620</td>\n",
       "      <td>A 32y old woman was referred to Yas hospital d...</td>\n",
       "      <td>A 32y old woman was referred to Yas hospital d...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 32y old woman was referred to Yas ...</td>\n",
       "      <td>267</td>\n",
       "      <td>0.958792</td>\n",
       "      <td>0.984211</td>\n",
       "      <td>0.968254</td>\n",
       "      <td>0.984211</td>\n",
       "      <td>0.984211</td>\n",
       "      <td>0.993698</td>\n",
       "      <td>10</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.956558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>723</th>\n",
       "      <td>89629</td>\n",
       "      <td>A 68-year-old man who was otherwise well, pres...</td>\n",
       "      <td>A 45-year-old man presented with a 6 month his...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you today?\\...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 68-year-old man who was otherwise ...</td>\n",
       "      <td>723</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.070395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>788</th>\n",
       "      <td>4868</td>\n",
       "      <td>A 71-year-old woman was referred for evaluatio...</td>\n",
       "      <td>A 71-year-old man presented with back pain and...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 71-year-old woman was referred for...</td>\n",
       "      <td>788</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.066993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>144030</td>\n",
       "      <td>A 69-year-old African American male with a pas...</td>\n",
       "      <td>A 56-year-old male with a history of hypertens...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you feeling...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'A 69-year-old African American male ...</td>\n",
       "      <td>363</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>0.064267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>424</th>\n",
       "      <td>111234</td>\n",
       "      <td>Our patient is an 8-year-old boy, first admitt...</td>\n",
       "      <td>A 7-year-old boy was admitted to our departmen...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'Our patient is an 8-year-old boy, fi...</td>\n",
       "      <td>424</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.063591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>770</th>\n",
       "      <td>9782</td>\n",
       "      <td>The patient was a 9-year-old boy with a histor...</td>\n",
       "      <td>A 9-year-old girl visited our hospital because...</td>\n",
       "      <td>Doctor: Hi, what brings you here today?\\nPatie...</td>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>{'gold': 'The patient was a 9-year-old boy wit...</td>\n",
       "      <td>770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.063591</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "699   86402  A 27-year-old female patient with good general...   \n",
       "832  108907  A 45-year-old male patient presented to us who...   \n",
       "197  115484  A 71-year-old female patient was referred to t...   \n",
       "342  139279  A 24-year-old female patient was referred to t...   \n",
       "267  202620  A 32y old woman was referred to Yas hospital d...   \n",
       "..      ...                                                ...   \n",
       "723   89629  A 68-year-old man who was otherwise well, pres...   \n",
       "788    4868  A 71-year-old woman was referred for evaluatio...   \n",
       "363  144030  A 69-year-old African American male with a pas...   \n",
       "424  111234  Our patient is an 8-year-old boy, first admitt...   \n",
       "770    9782  The patient was a 9-year-old boy with a histor...   \n",
       "\n",
       "                                                  pred  \\\n",
       "699  A 25-year-old female patient in good general h...   \n",
       "832  A 45-year-old male patient presented to the cl...   \n",
       "197  A 54-year-old female patient with a giant abdo...   \n",
       "342  A 28-year-old female patient was referred to t...   \n",
       "267  A 32y old woman was referred to Yas hospital d...   \n",
       "..                                                 ...   \n",
       "723  A 45-year-old man presented with a 6 month his...   \n",
       "788  A 71-year-old man presented with back pain and...   \n",
       "363  A 56-year-old male with a history of hypertens...   \n",
       "424  A 7-year-old boy was admitted to our departmen...   \n",
       "770  A 9-year-old girl visited our hospital because...   \n",
       "\n",
       "                                          conversation  \\\n",
       "699  Doctor: Good morning, how are you today?\\nPati...   \n",
       "832  Doctor: Good morning, sir. What brings you her...   \n",
       "197  Doctor: Good morning, how are you feeling toda...   \n",
       "342  Doctor: Hello, How are you feeling today?\\nPat...   \n",
       "267  Doctor: Hello, how are you feeling today?\\nPat...   \n",
       "..                                                 ...   \n",
       "723  Doctor: Good morning, sir. How are you today?\\...   \n",
       "788  Doctor: Hi, how are you feeling today?\\nPatien...   \n",
       "363  Doctor: Good morning, sir. How are you feeling...   \n",
       "424  Doctor: Good morning, what brings you here tod...   \n",
       "770  Doctor: Hi, what brings you here today?\\nPatie...   \n",
       "\n",
       "                    model_name  \\\n",
       "699  meditron-13b-direct-trunc   \n",
       "832  meditron-13b-direct-trunc   \n",
       "197  meditron-13b-direct-trunc   \n",
       "342  meditron-13b-direct-trunc   \n",
       "267  meditron-13b-direct-trunc   \n",
       "..                         ...   \n",
       "723  meditron-13b-direct-trunc   \n",
       "788  meditron-13b-direct-trunc   \n",
       "363  meditron-13b-direct-trunc   \n",
       "424  meditron-13b-direct-trunc   \n",
       "770  meditron-13b-direct-trunc   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "699  {'gold': 'A 27-year-old female patient with go...   699  0.960636   \n",
       "832  {'gold': 'A 45-year-old male patient presented...   832  0.995927   \n",
       "197  {'gold': 'A 71-year-old female patient was ref...   197  0.987310   \n",
       "342  {'gold': 'A 24-year-old female patient was ref...   342  0.974654   \n",
       "267  {'gold': 'A 32y old woman was referred to Yas ...   267  0.958792   \n",
       "..                                                 ...   ...       ...   \n",
       "723  {'gold': 'A 68-year-old man who was otherwise ...   723  0.000000   \n",
       "788  {'gold': 'A 71-year-old woman was referred for...   788  0.000000   \n",
       "363  {'gold': 'A 69-year-old African American male ...   363  0.000000   \n",
       "424  {'gold': 'Our patient is an 8-year-old boy, fi...   424  0.000000   \n",
       "770  {'gold': 'The patient was a 9-year-old boy wit...   770  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "699  0.995455  0.990868  0.995455   0.995455  0.996016                    10   \n",
       "832  1.000000  1.000000  1.000000   1.000000  0.999406                    10   \n",
       "197  0.990909  0.986301  0.990909   0.990909  0.997629                    10   \n",
       "342  0.992806  0.987952  0.992806   0.992806  0.992954                     9   \n",
       "267  0.984211  0.968254  0.984211   0.984211  0.993698                    10   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "723  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "788  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "363  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "424  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "770  0.000000  0.000000  0.000000   0.000000  0.000000                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "699                  8                    9                    10   \n",
       "832                  7                    7                    10   \n",
       "197                  7                    9                     8   \n",
       "342                  7                    8                    10   \n",
       "267                  7                    6                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "723                  7                    8                    10   \n",
       "788                  7                    7                    10   \n",
       "363                  8                    6                     9   \n",
       "424                  7                    6                    10   \n",
       "770                  7                    6                    10   \n",
       "\n",
       "     aggregated_score  \n",
       "699          0.981765  \n",
       "832          0.978919  \n",
       "197          0.971239  \n",
       "342          0.970078  \n",
       "267          0.956558  \n",
       "..                ...  \n",
       "723          0.070395  \n",
       "788          0.066993  \n",
       "363          0.064267  \n",
       "424          0.063591  \n",
       "770          0.063591  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-7b-generator\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>151451</td>\n",
       "      <td>A 16-year-old boy presented at the outpatient ...</td>\n",
       "      <td>A 30-year-old male patient presented with cont...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 16-year-old boy presented at the o...</td>\n",
       "      <td>494</td>\n",
       "      <td>0.637925</td>\n",
       "      <td>0.826966</td>\n",
       "      <td>0.731377</td>\n",
       "      <td>0.795506</td>\n",
       "      <td>0.795506</td>\n",
       "      <td>0.941360</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0.748490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>633</th>\n",
       "      <td>6721</td>\n",
       "      <td>A 76-year-old female patient was admitted to o...</td>\n",
       "      <td>A 76-year-old female patient was admitted to t...</td>\n",
       "      <td>Doctor: Good morning, I am Dr. X. How are you ...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 76-year-old female patient was adm...</td>\n",
       "      <td>633</td>\n",
       "      <td>0.599475</td>\n",
       "      <td>0.752747</td>\n",
       "      <td>0.674033</td>\n",
       "      <td>0.747253</td>\n",
       "      <td>0.747253</td>\n",
       "      <td>0.926569</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.733505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>214</th>\n",
       "      <td>191128</td>\n",
       "      <td>A 46-year-old Bahraini female diagnosed as pre...</td>\n",
       "      <td>A 44-year-old female was first admitted to the...</td>\n",
       "      <td>Doctor: Good morning! How can I help you today...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 46-year-old Bahraini female diagno...</td>\n",
       "      <td>214</td>\n",
       "      <td>0.592812</td>\n",
       "      <td>0.822115</td>\n",
       "      <td>0.695652</td>\n",
       "      <td>0.764423</td>\n",
       "      <td>0.764423</td>\n",
       "      <td>0.934824</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0.733438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>157160</td>\n",
       "      <td>A 12-year-old male child got admitted in our h...</td>\n",
       "      <td>An 18-year-old female patient, a known case of...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 12-year-old male child got admitte...</td>\n",
       "      <td>748</td>\n",
       "      <td>0.632372</td>\n",
       "      <td>0.823529</td>\n",
       "      <td>0.694087</td>\n",
       "      <td>0.757033</td>\n",
       "      <td>0.757033</td>\n",
       "      <td>0.941218</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>0.733300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>93128</td>\n",
       "      <td>A 59-year-old Greek man of Caucasian origin pr...</td>\n",
       "      <td>A 60-year-old female, non-smoker in excellent ...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 59-year-old Greek man of Caucasian...</td>\n",
       "      <td>573</td>\n",
       "      <td>0.636539</td>\n",
       "      <td>0.734043</td>\n",
       "      <td>0.684492</td>\n",
       "      <td>0.702128</td>\n",
       "      <td>0.702128</td>\n",
       "      <td>0.907087</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>0.725507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>589</th>\n",
       "      <td>6138</td>\n",
       "      <td>A 56-year-old man was admitted to a local hosp...</td>\n",
       "      <td>A 56-year-old man was admitted to a local hosp...</td>\n",
       "      <td>1. Doctor: Good morning, what brings you here ...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 56-year-old man was admitted to a ...</td>\n",
       "      <td>589</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.144638</td>\n",
       "      <td>0.010025</td>\n",
       "      <td>0.074813</td>\n",
       "      <td>0.074813</td>\n",
       "      <td>0.757643</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>0.120373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>650</th>\n",
       "      <td>62406</td>\n",
       "      <td>A 62-year-old man underwent a total thyroidect...</td>\n",
       "      <td>A 53-year-old woman underwent a total thyroide...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 62-year-old man underwent a total ...</td>\n",
       "      <td>650</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.113208</td>\n",
       "      <td>0.014218</td>\n",
       "      <td>0.066038</td>\n",
       "      <td>0.066038</td>\n",
       "      <td>0.759605</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>0.113819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>134393</td>\n",
       "      <td>A 30-years-old male with no comorbidities pres...</td>\n",
       "      <td>A 25-year-old male patient presented to our cl...</td>\n",
       "      <td>Doctor: Good morning. How are you feeling toda...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 30-years-old male with no comorbid...</td>\n",
       "      <td>660</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.091787</td>\n",
       "      <td>0.009709</td>\n",
       "      <td>0.048309</td>\n",
       "      <td>0.048309</td>\n",
       "      <td>0.761785</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0.112920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>140848</td>\n",
       "      <td>A 66-year-old male patient arrived at the emer...</td>\n",
       "      <td>A 61-year-old male patient was accidentally hi...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you feeling...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 66-year-old male patient arrived a...</td>\n",
       "      <td>537</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.075188</td>\n",
       "      <td>0.005038</td>\n",
       "      <td>0.055138</td>\n",
       "      <td>0.055138</td>\n",
       "      <td>0.765140</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0.112224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>223</th>\n",
       "      <td>99880</td>\n",
       "      <td>A 52-year-old man was referred to our dermatol...</td>\n",
       "      <td>A 55-year-old man presented with a history of ...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you today?\\...</td>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>{'gold': 'A 52-year-old man was referred to ou...</td>\n",
       "      <td>223</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.108247</td>\n",
       "      <td>0.005181</td>\n",
       "      <td>0.077320</td>\n",
       "      <td>0.077320</td>\n",
       "      <td>0.744114</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>0.106565</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "494  151451  A 16-year-old boy presented at the outpatient ...   \n",
       "633    6721  A 76-year-old female patient was admitted to o...   \n",
       "214  191128  A 46-year-old Bahraini female diagnosed as pre...   \n",
       "748  157160  A 12-year-old male child got admitted in our h...   \n",
       "573   93128  A 59-year-old Greek man of Caucasian origin pr...   \n",
       "..      ...                                                ...   \n",
       "589    6138  A 56-year-old man was admitted to a local hosp...   \n",
       "650   62406  A 62-year-old man underwent a total thyroidect...   \n",
       "660  134393  A 30-years-old male with no comorbidities pres...   \n",
       "537  140848  A 66-year-old male patient arrived at the emer...   \n",
       "223   99880  A 52-year-old man was referred to our dermatol...   \n",
       "\n",
       "                                                  pred  \\\n",
       "494  A 30-year-old male patient presented with cont...   \n",
       "633  A 76-year-old female patient was admitted to t...   \n",
       "214  A 44-year-old female was first admitted to the...   \n",
       "748  An 18-year-old female patient, a known case of...   \n",
       "573  A 60-year-old female, non-smoker in excellent ...   \n",
       "..                                                 ...   \n",
       "589  A 56-year-old man was admitted to a local hosp...   \n",
       "650  A 53-year-old woman underwent a total thyroide...   \n",
       "660  A 25-year-old male patient presented to our cl...   \n",
       "537  A 61-year-old male patient was accidentally hi...   \n",
       "223  A 55-year-old man presented with a history of ...   \n",
       "\n",
       "                                          conversation             model_name  \\\n",
       "494  Doctor: Good morning, what brings you here tod...  meditron-7b-generator   \n",
       "633  Doctor: Good morning, I am Dr. X. How are you ...  meditron-7b-generator   \n",
       "214  Doctor: Good morning! How can I help you today...  meditron-7b-generator   \n",
       "748  Doctor: Good morning, how are you feeling toda...  meditron-7b-generator   \n",
       "573  Doctor: Hello, how are you feeling today?\\nPat...  meditron-7b-generator   \n",
       "..                                                 ...                    ...   \n",
       "589  1. Doctor: Good morning, what brings you here ...  meditron-7b-generator   \n",
       "650  Doctor: Hello, how are you feeling today?\\nPat...  meditron-7b-generator   \n",
       "660  Doctor: Good morning. How are you feeling toda...  meditron-7b-generator   \n",
       "537  Doctor: Good morning, sir. How are you feeling...  meditron-7b-generator   \n",
       "223  Doctor: Good morning, sir. How are you today?\\...  meditron-7b-generator   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "494  {'gold': 'A 16-year-old boy presented at the o...   494  0.637925   \n",
       "633  {'gold': 'A 76-year-old female patient was adm...   633  0.599475   \n",
       "214  {'gold': 'A 46-year-old Bahraini female diagno...   214  0.592812   \n",
       "748  {'gold': 'A 12-year-old male child got admitte...   748  0.632372   \n",
       "573  {'gold': 'A 59-year-old Greek man of Caucasian...   573  0.636539   \n",
       "..                                                 ...   ...       ...   \n",
       "589  {'gold': 'A 56-year-old man was admitted to a ...   589  0.000000   \n",
       "650  {'gold': 'A 62-year-old man underwent a total ...   650  0.000000   \n",
       "660  {'gold': 'A 30-years-old male with no comorbid...   660  0.000000   \n",
       "537  {'gold': 'A 66-year-old male patient arrived a...   537  0.000000   \n",
       "223  {'gold': 'A 52-year-old man was referred to ou...   223  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "494  0.826966  0.731377  0.795506   0.795506  0.941360                     3   \n",
       "633  0.752747  0.674033  0.747253   0.747253  0.926569                     8   \n",
       "214  0.822115  0.695652  0.764423   0.764423  0.934824                     7   \n",
       "748  0.823529  0.694087  0.757033   0.757033  0.941218                     9   \n",
       "573  0.734043  0.684492  0.702128   0.702128  0.907087                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "589  0.144638  0.010025  0.074813   0.074813  0.757643                     1   \n",
       "650  0.113208  0.014218  0.066038   0.066038  0.759605                     1   \n",
       "660  0.091787  0.009709  0.048309   0.048309  0.761785                     1   \n",
       "537  0.075188  0.005038  0.055138   0.055138  0.765140                     1   \n",
       "223  0.108247  0.005181  0.077320   0.077320  0.744114                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "494                  6                    7                     9   \n",
       "633                  8                    8                    10   \n",
       "214                  7                    7                     6   \n",
       "748                  8                    7                     2   \n",
       "573                  9                    9                     9   \n",
       "..                 ...                  ...                   ...   \n",
       "589                  5                    5                     8   \n",
       "650                  6                    5                     7   \n",
       "660                  7                    7                     7   \n",
       "537                  8                    6                     7   \n",
       "223                  6                    5                     4   \n",
       "\n",
       "     aggregated_score  \n",
       "494          0.748490  \n",
       "633          0.733505  \n",
       "214          0.733438  \n",
       "748          0.733300  \n",
       "573          0.725507  \n",
       "..                ...  \n",
       "589          0.120373  \n",
       "650          0.113819  \n",
       "660          0.112920  \n",
       "537          0.112224  \n",
       "223          0.106565  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-13b-generator\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>97049</td>\n",
       "      <td>A 47-year-old cocaine-addicted man with a hist...</td>\n",
       "      <td>A 55-year-old man was admitted to the emergenc...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you feeling...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 47-year-old cocaine-addicted man w...</td>\n",
       "      <td>496</td>\n",
       "      <td>0.650960</td>\n",
       "      <td>0.892308</td>\n",
       "      <td>0.737113</td>\n",
       "      <td>0.861538</td>\n",
       "      <td>0.861538</td>\n",
       "      <td>0.952268</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.812166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568</th>\n",
       "      <td>87643</td>\n",
       "      <td>A 72-year-old woman with liver cirrhosis confi...</td>\n",
       "      <td>A 69-year-old female patient with liver cirrho...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 72-year-old woman with liver cirrh...</td>\n",
       "      <td>568</td>\n",
       "      <td>0.500491</td>\n",
       "      <td>0.824096</td>\n",
       "      <td>0.648910</td>\n",
       "      <td>0.761446</td>\n",
       "      <td>0.761446</td>\n",
       "      <td>0.926586</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.739505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>406</th>\n",
       "      <td>170531</td>\n",
       "      <td>We report a case of 38-year-old Greek woman wh...</td>\n",
       "      <td>A 55-year-old female patient was admitted to o...</td>\n",
       "      <td>Doctor: Good afternoon, how can I help you tod...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'We report a case of 38-year-old Gree...</td>\n",
       "      <td>406</td>\n",
       "      <td>0.541004</td>\n",
       "      <td>0.762115</td>\n",
       "      <td>0.650442</td>\n",
       "      <td>0.704846</td>\n",
       "      <td>0.704846</td>\n",
       "      <td>0.918477</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.701040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>976</th>\n",
       "      <td>134393</td>\n",
       "      <td>A 30-years-old male with no comorbidities pres...</td>\n",
       "      <td>A 25-year-old male patient, with no comorbidit...</td>\n",
       "      <td>Doctor: Good morning. How are you feeling toda...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 30-years-old male with no comorbid...</td>\n",
       "      <td>976</td>\n",
       "      <td>0.436288</td>\n",
       "      <td>0.764858</td>\n",
       "      <td>0.602597</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.911485</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.677276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>77341</td>\n",
       "      <td>A 59-year-old male with end-stage renal diseas...</td>\n",
       "      <td>A 63-year-old man with end-stage renal disease...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 59-year-old male with end-stage re...</td>\n",
       "      <td>816</td>\n",
       "      <td>0.513387</td>\n",
       "      <td>0.764846</td>\n",
       "      <td>0.596659</td>\n",
       "      <td>0.655582</td>\n",
       "      <td>0.655582</td>\n",
       "      <td>0.913561</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.670972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>141880</td>\n",
       "      <td>A 50-year-old male underwent a decompressive c...</td>\n",
       "      <td>A 45-year-old man was involved in a motor vehi...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you feeling...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 50-year-old male underwent a decom...</td>\n",
       "      <td>728</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.095477</td>\n",
       "      <td>0.010101</td>\n",
       "      <td>0.055276</td>\n",
       "      <td>0.055276</td>\n",
       "      <td>0.758935</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.124750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>181590</td>\n",
       "      <td>The patient was a 50-year-old male, 173 cm tal...</td>\n",
       "      <td>A 61-year-old man (height, 173 cm; weight, 85 ...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'The patient was a 50-year-old male, ...</td>\n",
       "      <td>178</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.047120</td>\n",
       "      <td>0.005263</td>\n",
       "      <td>0.041885</td>\n",
       "      <td>0.041885</td>\n",
       "      <td>0.726798</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0.114293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>182087</td>\n",
       "      <td>A 57-year-old female patient presented in our ...</td>\n",
       "      <td>A 53-year-old female patient presented to our ...</td>\n",
       "      <td>Doctor: Hello, how can I help you today?\\nPati...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'A 57-year-old female patient present...</td>\n",
       "      <td>196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.086538</td>\n",
       "      <td>0.014493</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.766511</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>0.112844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>920</th>\n",
       "      <td>14867</td>\n",
       "      <td>We describe the case of a 67-year-old man who ...</td>\n",
       "      <td>A previously healthy 16-year-old girl presente...</td>\n",
       "      <td>Doctor: Good morning, what brings you in today...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'We describe the case of a 67-year-ol...</td>\n",
       "      <td>920</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.077720</td>\n",
       "      <td>0.005208</td>\n",
       "      <td>0.051813</td>\n",
       "      <td>0.051813</td>\n",
       "      <td>0.736250</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>0.099159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>30959</td>\n",
       "      <td>The patient is a 12-year-old male from a rural...</td>\n",
       "      <td>A 10-year-old girl from rural Honduras present...</td>\n",
       "      <td>Doctor: Good morning, what brings you to the h...</td>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>{'gold': 'The patient is a 12-year-old male fr...</td>\n",
       "      <td>740</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.048485</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.024242</td>\n",
       "      <td>0.024242</td>\n",
       "      <td>0.731311</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0.089798</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "496   97049  A 47-year-old cocaine-addicted man with a hist...   \n",
       "568   87643  A 72-year-old woman with liver cirrhosis confi...   \n",
       "406  170531  We report a case of 38-year-old Greek woman wh...   \n",
       "976  134393  A 30-years-old male with no comorbidities pres...   \n",
       "816   77341  A 59-year-old male with end-stage renal diseas...   \n",
       "..      ...                                                ...   \n",
       "728  141880  A 50-year-old male underwent a decompressive c...   \n",
       "178  181590  The patient was a 50-year-old male, 173 cm tal...   \n",
       "196  182087  A 57-year-old female patient presented in our ...   \n",
       "920   14867  We describe the case of a 67-year-old man who ...   \n",
       "740   30959  The patient is a 12-year-old male from a rural...   \n",
       "\n",
       "                                                  pred  \\\n",
       "496  A 55-year-old man was admitted to the emergenc...   \n",
       "568  A 69-year-old female patient with liver cirrho...   \n",
       "406  A 55-year-old female patient was admitted to o...   \n",
       "976  A 25-year-old male patient, with no comorbidit...   \n",
       "816  A 63-year-old man with end-stage renal disease...   \n",
       "..                                                 ...   \n",
       "728  A 45-year-old man was involved in a motor vehi...   \n",
       "178  A 61-year-old man (height, 173 cm; weight, 85 ...   \n",
       "196  A 53-year-old female patient presented to our ...   \n",
       "920  A previously healthy 16-year-old girl presente...   \n",
       "740  A 10-year-old girl from rural Honduras present...   \n",
       "\n",
       "                                          conversation  \\\n",
       "496  Doctor: Good morning, sir. How are you feeling...   \n",
       "568  Doctor: Good morning, how are you feeling toda...   \n",
       "406  Doctor: Good afternoon, how can I help you tod...   \n",
       "976  Doctor: Good morning. How are you feeling toda...   \n",
       "816  Doctor: Good morning, how are you feeling toda...   \n",
       "..                                                 ...   \n",
       "728  Doctor: Good morning, sir. How are you feeling...   \n",
       "178  Doctor: Hello, how are you feeling today?\\nPat...   \n",
       "196  Doctor: Hello, how can I help you today?\\nPati...   \n",
       "920  Doctor: Good morning, what brings you in today...   \n",
       "740  Doctor: Good morning, what brings you to the h...   \n",
       "\n",
       "                 model_name  \\\n",
       "496  meditron-13b-generator   \n",
       "568  meditron-13b-generator   \n",
       "406  meditron-13b-generator   \n",
       "976  meditron-13b-generator   \n",
       "816  meditron-13b-generator   \n",
       "..                      ...   \n",
       "728  meditron-13b-generator   \n",
       "178  meditron-13b-generator   \n",
       "196  meditron-13b-generator   \n",
       "920  meditron-13b-generator   \n",
       "740  meditron-13b-generator   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "496  {'gold': 'A 47-year-old cocaine-addicted man w...   496  0.650960   \n",
       "568  {'gold': 'A 72-year-old woman with liver cirrh...   568  0.500491   \n",
       "406  {'gold': 'We report a case of 38-year-old Gree...   406  0.541004   \n",
       "976  {'gold': 'A 30-years-old male with no comorbid...   976  0.436288   \n",
       "816  {'gold': 'A 59-year-old male with end-stage re...   816  0.513387   \n",
       "..                                                 ...   ...       ...   \n",
       "728  {'gold': 'A 50-year-old male underwent a decom...   728  0.000000   \n",
       "178  {'gold': 'The patient was a 50-year-old male, ...   178  0.000000   \n",
       "196  {'gold': 'A 57-year-old female patient present...   196  0.000000   \n",
       "920  {'gold': 'We describe the case of a 67-year-ol...   920  0.000000   \n",
       "740  {'gold': 'The patient is a 12-year-old male fr...   740  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "496  0.892308  0.737113  0.861538   0.861538  0.952268                     9   \n",
       "568  0.824096  0.648910  0.761446   0.761446  0.926586                     7   \n",
       "406  0.762115  0.650442  0.704846   0.704846  0.918477                     8   \n",
       "976  0.764858  0.602597  0.666667   0.666667  0.911485                     8   \n",
       "816  0.764846  0.596659  0.655582   0.655582  0.913561                     8   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "728  0.095477  0.010101  0.055276   0.055276  0.758935                     1   \n",
       "178  0.047120  0.005263  0.041885   0.041885  0.726798                     1   \n",
       "196  0.086538  0.014493  0.057692   0.057692  0.766511                     1   \n",
       "920  0.077720  0.005208  0.051813   0.051813  0.736250                     1   \n",
       "740  0.048485  0.000000  0.024242   0.024242  0.731311                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "496                  6                    7                    10   \n",
       "568                  9                    9                    10   \n",
       "406                  7                    7                    10   \n",
       "976                  8                    9                    10   \n",
       "816                  8                    7                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "728                  7                    7                    10   \n",
       "178                  8                    8                     9   \n",
       "196                  5                    7                     8   \n",
       "920                  7                    7                     3   \n",
       "740                  6                    6                     8   \n",
       "\n",
       "     aggregated_score  \n",
       "496          0.812166  \n",
       "568          0.739505  \n",
       "406          0.701040  \n",
       "976          0.677276  \n",
       "816          0.670972  \n",
       "..                ...  \n",
       "728          0.124750  \n",
       "178          0.114293  \n",
       "196          0.112844  \n",
       "920          0.099159  \n",
       "740          0.089798  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "llama-2-7b-direct\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>824</th>\n",
       "      <td>134359</td>\n",
       "      <td>A 50-year-old female presented to us with a 3-...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: [Insert Patien...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'A 50-year-old female presented to us...</td>\n",
       "      <td>824</td>\n",
       "      <td>0.469331</td>\n",
       "      <td>0.784648</td>\n",
       "      <td>0.659529</td>\n",
       "      <td>0.771855</td>\n",
       "      <td>0.771855</td>\n",
       "      <td>0.912624</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.744606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>943</th>\n",
       "      <td>88065</td>\n",
       "      <td>A 45-year-old male patient admitted on 17th Oc...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: John Doe\\nDate...</td>\n",
       "      <td>Doctor: Good morning, sir. Can you tell me wha...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'A 45-year-old male patient admitted ...</td>\n",
       "      <td>943</td>\n",
       "      <td>0.474956</td>\n",
       "      <td>0.827586</td>\n",
       "      <td>0.670996</td>\n",
       "      <td>0.775862</td>\n",
       "      <td>0.775862</td>\n",
       "      <td>0.903559</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.738305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>771</th>\n",
       "      <td>107993</td>\n",
       "      <td>A 7-year-old boy, who was a victim of a road t...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: [Insert Patien...</td>\n",
       "      <td>Doctor: Hello, How are you feeling today?\\nPat...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'A 7-year-old boy, who was a victim o...</td>\n",
       "      <td>771</td>\n",
       "      <td>0.487494</td>\n",
       "      <td>0.781857</td>\n",
       "      <td>0.685466</td>\n",
       "      <td>0.760259</td>\n",
       "      <td>0.760259</td>\n",
       "      <td>0.911034</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.733093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>574</th>\n",
       "      <td>82043</td>\n",
       "      <td>A 4-month-old Chinese baby boy had experienced...</td>\n",
       "      <td>Clinical Note:\\nPatient: [Name of Patient]\\n...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'A 4-month-old Chinese baby boy had e...</td>\n",
       "      <td>574</td>\n",
       "      <td>0.535470</td>\n",
       "      <td>0.756098</td>\n",
       "      <td>0.656863</td>\n",
       "      <td>0.721951</td>\n",
       "      <td>0.721951</td>\n",
       "      <td>0.898434</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.724919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>17228</td>\n",
       "      <td>An eleven-day-old female infant was admitted t...</td>\n",
       "      <td>Clinical Note:\\nPatient: [Infant's name]\\nCh...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'An eleven-day-old female infant was ...</td>\n",
       "      <td>431</td>\n",
       "      <td>0.477819</td>\n",
       "      <td>0.811966</td>\n",
       "      <td>0.673820</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>0.730769</td>\n",
       "      <td>0.920028</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.722837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>983</th>\n",
       "      <td>47804</td>\n",
       "      <td>This case is that of a 49-year-old woman origi...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: [Insert Patien...</td>\n",
       "      <td>Doctor: Good morning, How can I help you today...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'This case is that of a 49-year-old w...</td>\n",
       "      <td>983</td>\n",
       "      <td>0.057702</td>\n",
       "      <td>0.170704</td>\n",
       "      <td>0.090173</td>\n",
       "      <td>0.119954</td>\n",
       "      <td>0.119954</td>\n",
       "      <td>0.781647</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>0.135661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>190880</td>\n",
       "      <td>A 37-year-old Caucasian male with a known hist...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: [Patient's nam...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'A 37-year-old Caucasian male with a ...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.028492</td>\n",
       "      <td>0.181354</td>\n",
       "      <td>0.058899</td>\n",
       "      <td>0.076628</td>\n",
       "      <td>0.076628</td>\n",
       "      <td>0.791449</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>0.130900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>131369</td>\n",
       "      <td>The first child was a 10-year-old girl who pre...</td>\n",
       "      <td>Clinical Note:\\nPatient: [Name of Patient]\\n...</td>\n",
       "      <td>Doctor: Hi, how are you doing today?\\nPatient:...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'The first child was a 10-year-old gi...</td>\n",
       "      <td>93</td>\n",
       "      <td>0.055378</td>\n",
       "      <td>0.203791</td>\n",
       "      <td>0.104513</td>\n",
       "      <td>0.104265</td>\n",
       "      <td>0.104265</td>\n",
       "      <td>0.796518</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.120370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>73652</td>\n",
       "      <td>We report the case of a 49-year-old woman with...</td>\n",
       "      <td>Clinical Note:\\nPatient Name: [Patient's nam...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'We report the case of a 49-year-old ...</td>\n",
       "      <td>47</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.184514</td>\n",
       "      <td>0.029752</td>\n",
       "      <td>0.069193</td>\n",
       "      <td>0.069193</td>\n",
       "      <td>0.778542</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.118451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>386</th>\n",
       "      <td>152597</td>\n",
       "      <td>The second case was a 70-year-old married woma...</td>\n",
       "      <td>Clinical Note:\\nPatient: John Doe\\nChief Com...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>{'gold': 'The second case was a 70-year-old ma...</td>\n",
       "      <td>386</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.099668</td>\n",
       "      <td>0.006689</td>\n",
       "      <td>0.046512</td>\n",
       "      <td>0.046512</td>\n",
       "      <td>0.734799</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0.112039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "824  134359  A 50-year-old female presented to us with a 3-...   \n",
       "943   88065  A 45-year-old male patient admitted on 17th Oc...   \n",
       "771  107993  A 7-year-old boy, who was a victim of a road t...   \n",
       "574   82043  A 4-month-old Chinese baby boy had experienced...   \n",
       "431   17228  An eleven-day-old female infant was admitted t...   \n",
       "..      ...                                                ...   \n",
       "983   47804  This case is that of a 49-year-old woman origi...   \n",
       "4    190880  A 37-year-old Caucasian male with a known hist...   \n",
       "93   131369  The first child was a 10-year-old girl who pre...   \n",
       "47    73652  We report the case of a 49-year-old woman with...   \n",
       "386  152597  The second case was a 70-year-old married woma...   \n",
       "\n",
       "                                                  pred  \\\n",
       "824    Clinical Note:\\nPatient Name: [Insert Patien...   \n",
       "943    Clinical Note:\\nPatient Name: John Doe\\nDate...   \n",
       "771    Clinical Note:\\nPatient Name: [Insert Patien...   \n",
       "574    Clinical Note:\\nPatient: [Name of Patient]\\n...   \n",
       "431    Clinical Note:\\nPatient: [Infant's name]\\nCh...   \n",
       "..                                                 ...   \n",
       "983    Clinical Note:\\nPatient Name: [Insert Patien...   \n",
       "4      Clinical Note:\\nPatient Name: [Patient's nam...   \n",
       "93     Clinical Note:\\nPatient: [Name of Patient]\\n...   \n",
       "47     Clinical Note:\\nPatient Name: [Patient's nam...   \n",
       "386    Clinical Note:\\nPatient: John Doe\\nChief Com...   \n",
       "\n",
       "                                          conversation         model_name  \\\n",
       "824  Doctor: Good morning, how are you feeling toda...  llama-2-7b-direct   \n",
       "943  Doctor: Good morning, sir. Can you tell me wha...  llama-2-7b-direct   \n",
       "771  Doctor: Hello, How are you feeling today?\\nPat...  llama-2-7b-direct   \n",
       "574  Doctor: Good morning, what brings you here tod...  llama-2-7b-direct   \n",
       "431  Doctor: Good morning, how are you feeling toda...  llama-2-7b-direct   \n",
       "..                                                 ...                ...   \n",
       "983  Doctor: Good morning, How can I help you today...  llama-2-7b-direct   \n",
       "4    Doctor: Hi, how are you feeling today?\\nPatien...  llama-2-7b-direct   \n",
       "93   Doctor: Hi, how are you doing today?\\nPatient:...  llama-2-7b-direct   \n",
       "47   Doctor: Good morning, how are you feeling toda...  llama-2-7b-direct   \n",
       "386  Doctor: Hello, how are you feeling today?\\nPat...  llama-2-7b-direct   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "824  {'gold': 'A 50-year-old female presented to us...   824  0.469331   \n",
       "943  {'gold': 'A 45-year-old male patient admitted ...   943  0.474956   \n",
       "771  {'gold': 'A 7-year-old boy, who was a victim o...   771  0.487494   \n",
       "574  {'gold': 'A 4-month-old Chinese baby boy had e...   574  0.535470   \n",
       "431  {'gold': 'An eleven-day-old female infant was ...   431  0.477819   \n",
       "..                                                 ...   ...       ...   \n",
       "983  {'gold': 'This case is that of a 49-year-old w...   983  0.057702   \n",
       "4    {'gold': 'A 37-year-old Caucasian male with a ...     4  0.028492   \n",
       "93   {'gold': 'The first child was a 10-year-old gi...    93  0.055378   \n",
       "47   {'gold': 'We report the case of a 49-year-old ...    47  0.000000   \n",
       "386  {'gold': 'The second case was a 70-year-old ma...   386  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "824  0.784648  0.659529  0.771855   0.771855  0.912624                     9   \n",
       "943  0.827586  0.670996  0.775862   0.775862  0.903559                     9   \n",
       "771  0.781857  0.685466  0.760259   0.760259  0.911034                     9   \n",
       "574  0.756098  0.656863  0.721951   0.721951  0.898434                     9   \n",
       "431  0.811966  0.673820  0.730769   0.730769  0.920028                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "983  0.170704  0.090173  0.119954   0.119954  0.781647                     1   \n",
       "4    0.181354  0.058899  0.076628   0.076628  0.791449                     2   \n",
       "93   0.203791  0.104513  0.104265   0.104265  0.796518                     1   \n",
       "47   0.184514  0.029752  0.069193   0.069193  0.778542                     1   \n",
       "386  0.099668  0.006689  0.046512   0.046512  0.734799                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "824                  9                    9                    10   \n",
       "943                  7                    6                    10   \n",
       "771                  7                    7                    10   \n",
       "574                  9                    9                    10   \n",
       "431                  7                    7                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "983                  1                    2                     6   \n",
       "4                    4                    2                     8   \n",
       "93                   1                    1                     1   \n",
       "47                   5                    6                     3   \n",
       "386                  7                    7                     7   \n",
       "\n",
       "     aggregated_score  \n",
       "824          0.744606  \n",
       "943          0.738305  \n",
       "771          0.733093  \n",
       "574          0.724919  \n",
       "431          0.722837  \n",
       "..                ...  \n",
       "983          0.135661  \n",
       "4            0.130900  \n",
       "93           0.120370  \n",
       "47           0.118451  \n",
       "386          0.112039  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "llama-2-13b-direct\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>125659</td>\n",
       "      <td>A 30-year-old woman was admitted to our instit...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Good morning, what brings you in today...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 30-year-old woman was admitted to ...</td>\n",
       "      <td>721</td>\n",
       "      <td>0.455920</td>\n",
       "      <td>0.783784</td>\n",
       "      <td>0.628959</td>\n",
       "      <td>0.711712</td>\n",
       "      <td>0.711712</td>\n",
       "      <td>0.898758</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.701758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>825</th>\n",
       "      <td>56661</td>\n",
       "      <td>The patient was a 56-year-old female with a hi...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Hello, my name is Dr. Smith. What brin...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'The patient was a 56-year-old female...</td>\n",
       "      <td>825</td>\n",
       "      <td>0.434766</td>\n",
       "      <td>0.775510</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.709184</td>\n",
       "      <td>0.709184</td>\n",
       "      <td>0.888582</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.697672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>730</th>\n",
       "      <td>108605</td>\n",
       "      <td>A 25-year-old primigravida presented to the an...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Good morning, how can I help you today...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 25-year-old primigravida presented...</td>\n",
       "      <td>730</td>\n",
       "      <td>0.349436</td>\n",
       "      <td>0.762115</td>\n",
       "      <td>0.579646</td>\n",
       "      <td>0.700441</td>\n",
       "      <td>0.700441</td>\n",
       "      <td>0.912628</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.678346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>417</th>\n",
       "      <td>90832</td>\n",
       "      <td>A 22-yr-old male, farmer by occupation was ref...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 22-yr-old male, farmer by occupati...</td>\n",
       "      <td>417</td>\n",
       "      <td>0.543377</td>\n",
       "      <td>0.743119</td>\n",
       "      <td>0.599078</td>\n",
       "      <td>0.688073</td>\n",
       "      <td>0.688073</td>\n",
       "      <td>0.910912</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0.668907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>912</th>\n",
       "      <td>81547</td>\n",
       "      <td>A 66-year-old woman of Middle-Eastern backgrou...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 66-year-old woman of Middle-Easter...</td>\n",
       "      <td>912</td>\n",
       "      <td>0.488971</td>\n",
       "      <td>0.786047</td>\n",
       "      <td>0.626168</td>\n",
       "      <td>0.609302</td>\n",
       "      <td>0.609302</td>\n",
       "      <td>0.901456</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.666993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>121315</td>\n",
       "      <td>A 57-year-old Japanese woman presented with ep...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Hello, I'm Dr. Smith. What brings you ...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 57-year-old Japanese woman present...</td>\n",
       "      <td>323</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042945</td>\n",
       "      <td>0.003077</td>\n",
       "      <td>0.033742</td>\n",
       "      <td>0.033742</td>\n",
       "      <td>0.744467</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>0.067524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>632</th>\n",
       "      <td>34897</td>\n",
       "      <td>A 64-year-old Caucasian woman presented for ro...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Hello, I'm Dr. Smith. How are you feel...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 64-year-old Caucasian woman presen...</td>\n",
       "      <td>632</td>\n",
       "      <td>0.015801</td>\n",
       "      <td>0.109005</td>\n",
       "      <td>0.033254</td>\n",
       "      <td>0.056872</td>\n",
       "      <td>0.056872</td>\n",
       "      <td>0.762508</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.067059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>859</th>\n",
       "      <td>185719</td>\n",
       "      <td>A 66-year-old man with a history of infiltrati...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 66-year-old man with a history of ...</td>\n",
       "      <td>859</td>\n",
       "      <td>0.024989</td>\n",
       "      <td>0.087649</td>\n",
       "      <td>0.026631</td>\n",
       "      <td>0.058433</td>\n",
       "      <td>0.058433</td>\n",
       "      <td>0.729973</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.062841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>83894</td>\n",
       "      <td>Informed patient was consent was obtained for ...</td>\n",
       "      <td>Sure! Here is the clinical note based on the...</td>\n",
       "      <td>Doctor: Good morning, Mr. X. How are you feeli...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'Informed patient was consent was obt...</td>\n",
       "      <td>960</td>\n",
       "      <td>0.008961</td>\n",
       "      <td>0.102508</td>\n",
       "      <td>0.017486</td>\n",
       "      <td>0.043621</td>\n",
       "      <td>0.043621</td>\n",
       "      <td>0.739844</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0.059544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>831</th>\n",
       "      <td>135418</td>\n",
       "      <td>A 44-year-old male was brought with a metallic...</td>\n",
       "      <td>Sure! Here is the corresponding clinical not...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>{'gold': 'A 44-year-old male was brought with ...</td>\n",
       "      <td>831</td>\n",
       "      <td>0.011622</td>\n",
       "      <td>0.073394</td>\n",
       "      <td>0.025287</td>\n",
       "      <td>0.034404</td>\n",
       "      <td>0.034404</td>\n",
       "      <td>0.722435</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.055302</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "721  125659  A 30-year-old woman was admitted to our instit...   \n",
       "825   56661  The patient was a 56-year-old female with a hi...   \n",
       "730  108605  A 25-year-old primigravida presented to the an...   \n",
       "417   90832  A 22-yr-old male, farmer by occupation was ref...   \n",
       "912   81547  A 66-year-old woman of Middle-Eastern backgrou...   \n",
       "..      ...                                                ...   \n",
       "323  121315  A 57-year-old Japanese woman presented with ep...   \n",
       "632   34897  A 64-year-old Caucasian woman presented for ro...   \n",
       "859  185719  A 66-year-old man with a history of infiltrati...   \n",
       "960   83894  Informed patient was consent was obtained for ...   \n",
       "831  135418  A 44-year-old male was brought with a metallic...   \n",
       "\n",
       "                                                  pred  \\\n",
       "721    Sure! Here is the corresponding clinical not...   \n",
       "825    Sure! Here is the corresponding clinical not...   \n",
       "730    Sure! Here is the corresponding clinical not...   \n",
       "417    Sure! Here is the corresponding clinical not...   \n",
       "912    Sure! Here is the corresponding clinical not...   \n",
       "..                                                 ...   \n",
       "323    Sure! Here is the corresponding clinical not...   \n",
       "632    Sure! Here is the corresponding clinical not...   \n",
       "859    Sure! Here is the corresponding clinical not...   \n",
       "960    Sure! Here is the clinical note based on the...   \n",
       "831    Sure! Here is the corresponding clinical not...   \n",
       "\n",
       "                                          conversation          model_name  \\\n",
       "721  Doctor: Good morning, what brings you in today...  llama-2-13b-direct   \n",
       "825  Doctor: Hello, my name is Dr. Smith. What brin...  llama-2-13b-direct   \n",
       "730  Doctor: Good morning, how can I help you today...  llama-2-13b-direct   \n",
       "417  Doctor: Good morning, what brings you here tod...  llama-2-13b-direct   \n",
       "912  Doctor: Good morning, what brings you here tod...  llama-2-13b-direct   \n",
       "..                                                 ...                 ...   \n",
       "323  Doctor: Hello, I'm Dr. Smith. What brings you ...  llama-2-13b-direct   \n",
       "632  Doctor: Hello, I'm Dr. Smith. How are you feel...  llama-2-13b-direct   \n",
       "859  Doctor: Good morning, how are you feeling toda...  llama-2-13b-direct   \n",
       "960  Doctor: Good morning, Mr. X. How are you feeli...  llama-2-13b-direct   \n",
       "831  Doctor: Hello, how are you feeling today?\\nPat...  llama-2-13b-direct   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "721  {'gold': 'A 30-year-old woman was admitted to ...   721  0.455920   \n",
       "825  {'gold': 'The patient was a 56-year-old female...   825  0.434766   \n",
       "730  {'gold': 'A 25-year-old primigravida presented...   730  0.349436   \n",
       "417  {'gold': 'A 22-yr-old male, farmer by occupati...   417  0.543377   \n",
       "912  {'gold': 'A 66-year-old woman of Middle-Easter...   912  0.488971   \n",
       "..                                                 ...   ...       ...   \n",
       "323  {'gold': 'A 57-year-old Japanese woman present...   323  0.000000   \n",
       "632  {'gold': 'A 64-year-old Caucasian woman presen...   632  0.015801   \n",
       "859  {'gold': 'A 66-year-old man with a history of ...   859  0.024989   \n",
       "960  {'gold': 'Informed patient was consent was obt...   960  0.008961   \n",
       "831  {'gold': 'A 44-year-old male was brought with ...   831  0.011622   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "721  0.783784  0.628959  0.711712   0.711712  0.898758                     8   \n",
       "825  0.775510  0.615385  0.709184   0.709184  0.888582                     7   \n",
       "730  0.762115  0.579646  0.700441   0.700441  0.912628                     9   \n",
       "417  0.743119  0.599078  0.688073   0.688073  0.910912                     3   \n",
       "912  0.786047  0.626168  0.609302   0.609302  0.901456                     7   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "323  0.042945  0.003077  0.033742   0.033742  0.744467                     2   \n",
       "632  0.109005  0.033254  0.056872   0.056872  0.762508                     1   \n",
       "859  0.087649  0.026631  0.058433   0.058433  0.729973                     1   \n",
       "960  0.102508  0.017486  0.043621   0.043621  0.739844                     1   \n",
       "831  0.073394  0.025287  0.034404   0.034404  0.722435                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "721                  8                    8                    10   \n",
       "825                  9                    9                    10   \n",
       "730                  8                    8                    10   \n",
       "417                  9                    8                     9   \n",
       "912                  9                    9                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "323                  2                    3                     6   \n",
       "632                  1                    1                     1   \n",
       "859                  1                    1                     1   \n",
       "960                  1                    2                     1   \n",
       "831                  1                    1                     3   \n",
       "\n",
       "     aggregated_score  \n",
       "721          0.701758  \n",
       "825          0.697672  \n",
       "730          0.678346  \n",
       "417          0.668907  \n",
       "912          0.666993  \n",
       "..                ...  \n",
       "323          0.067524  \n",
       "632          0.067059  \n",
       "859          0.062841  \n",
       "960          0.059544  \n",
       "831          0.055302  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "mistral-7b-direct\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>108896</td>\n",
       "      <td>A 37-year-old Caucasian woman was admitted to ...</td>\n",
       "      <td>\\nPatient Name: Sarah\\nDate of Admission: Not ...</td>\n",
       "      <td>Doctor: Hi there, my name is Dr. X. Can you te...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 37-year-old Caucasian woman was ad...</td>\n",
       "      <td>93</td>\n",
       "      <td>0.446003</td>\n",
       "      <td>0.841346</td>\n",
       "      <td>0.700483</td>\n",
       "      <td>0.802885</td>\n",
       "      <td>0.802885</td>\n",
       "      <td>0.918869</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.762323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>452</th>\n",
       "      <td>140682</td>\n",
       "      <td>A female child aged 12 years, initially presen...</td>\n",
       "      <td>\\nClinical Note:\\nPatient: 12-year-old female ...</td>\n",
       "      <td>Doctor: Hi, I am Dr. X. How can I help you tod...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A female child aged 12 years, initia...</td>\n",
       "      <td>452</td>\n",
       "      <td>0.489894</td>\n",
       "      <td>0.837529</td>\n",
       "      <td>0.735632</td>\n",
       "      <td>0.810069</td>\n",
       "      <td>0.810069</td>\n",
       "      <td>0.899905</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>0.754025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>689</th>\n",
       "      <td>6628</td>\n",
       "      <td>A 77-year-old male with a past medical history...</td>\n",
       "      <td>\\nPatient: 72-year-old male with a history of ...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 77-year-old male with a past medic...</td>\n",
       "      <td>689</td>\n",
       "      <td>0.550962</td>\n",
       "      <td>0.808399</td>\n",
       "      <td>0.670185</td>\n",
       "      <td>0.766404</td>\n",
       "      <td>0.766404</td>\n",
       "      <td>0.867168</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>0.728508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>130604</td>\n",
       "      <td>A 32-year-old male presented to the emergency ...</td>\n",
       "      <td>\\nPatient: 39-year-old male with a history of ...</td>\n",
       "      <td>Doctor: Good morning, what brings you to the e...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 32-year-old male presented to the ...</td>\n",
       "      <td>302</td>\n",
       "      <td>0.437850</td>\n",
       "      <td>0.817708</td>\n",
       "      <td>0.628272</td>\n",
       "      <td>0.760417</td>\n",
       "      <td>0.760417</td>\n",
       "      <td>0.919878</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.718136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>186100</td>\n",
       "      <td>A 24-year-old female patient presented to the ...</td>\n",
       "      <td>\\nPatient: [Name]\\nDate of Visit: [Date]\\nReas...</td>\n",
       "      <td>Doctor: Hello, how can I help you today?\\nPati...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 24-year-old female patient present...</td>\n",
       "      <td>464</td>\n",
       "      <td>0.471555</td>\n",
       "      <td>0.794444</td>\n",
       "      <td>0.653631</td>\n",
       "      <td>0.738889</td>\n",
       "      <td>0.738889</td>\n",
       "      <td>0.915943</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>0.716090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>971</th>\n",
       "      <td>47995</td>\n",
       "      <td>A 36-year-old female, Gravida 2, Para 1, at 21...</td>\n",
       "      <td>\\nPatient: Gravida 2, Para 1, at 21 weeks of p...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 36-year-old female, Gravida 2, Par...</td>\n",
       "      <td>971</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009346</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009346</td>\n",
       "      <td>0.009346</td>\n",
       "      <td>0.797113</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.106051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>34124</td>\n",
       "      <td>A 74-year old male with a history of diabetes ...</td>\n",
       "      <td>\\nPatient: [Name], a 65-year-old male with a h...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 74-year old male with a history of...</td>\n",
       "      <td>103</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.019512</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.019512</td>\n",
       "      <td>0.019512</td>\n",
       "      <td>0.656015</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.104837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>846</th>\n",
       "      <td>92205</td>\n",
       "      <td>A 25-year-old Caucasian female underwent a spl...</td>\n",
       "      <td>\\nPatient: 37-year-old female with a history o...</td>\n",
       "      <td>Doctor: Hello, I'm Dr. X. What brings you here...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 25-year-old Caucasian female under...</td>\n",
       "      <td>846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017857</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017857</td>\n",
       "      <td>0.017857</td>\n",
       "      <td>0.799412</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0.088427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>760</th>\n",
       "      <td>42883</td>\n",
       "      <td>A 29-year-old woman, gravida 1, para 0, was ad...</td>\n",
       "      <td>\\nPatient: Ms. Smith, 32 years old, G1P0, pres...</td>\n",
       "      <td>Doctor: Good morning, I am Dr. Johnson. How ar...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 29-year-old woman, gravida 1, para...</td>\n",
       "      <td>760</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017778</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008889</td>\n",
       "      <td>0.008889</td>\n",
       "      <td>0.809999</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.083226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>114194</td>\n",
       "      <td>A 26-year-old male patient was suffering from ...</td>\n",
       "      <td>\\nPatient: [Name]\\nDate of Visit: [Date]\\nReas...</td>\n",
       "      <td>Doctor: Hello, how are you today?\\nPatient: I'...</td>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>{'gold': 'A 26-year-old male patient was suffe...</td>\n",
       "      <td>100</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.019481</td>\n",
       "      <td>0.688761</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0.064298</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "93   108896  A 37-year-old Caucasian woman was admitted to ...   \n",
       "452  140682  A female child aged 12 years, initially presen...   \n",
       "689    6628  A 77-year-old male with a past medical history...   \n",
       "302  130604  A 32-year-old male presented to the emergency ...   \n",
       "464  186100  A 24-year-old female patient presented to the ...   \n",
       "..      ...                                                ...   \n",
       "971   47995  A 36-year-old female, Gravida 2, Para 1, at 21...   \n",
       "103   34124  A 74-year old male with a history of diabetes ...   \n",
       "846   92205  A 25-year-old Caucasian female underwent a spl...   \n",
       "760   42883  A 29-year-old woman, gravida 1, para 0, was ad...   \n",
       "100  114194  A 26-year-old male patient was suffering from ...   \n",
       "\n",
       "                                                  pred  \\\n",
       "93   \\nPatient Name: Sarah\\nDate of Admission: Not ...   \n",
       "452  \\nClinical Note:\\nPatient: 12-year-old female ...   \n",
       "689  \\nPatient: 72-year-old male with a history of ...   \n",
       "302  \\nPatient: 39-year-old male with a history of ...   \n",
       "464  \\nPatient: [Name]\\nDate of Visit: [Date]\\nReas...   \n",
       "..                                                 ...   \n",
       "971  \\nPatient: Gravida 2, Para 1, at 21 weeks of p...   \n",
       "103  \\nPatient: [Name], a 65-year-old male with a h...   \n",
       "846  \\nPatient: 37-year-old female with a history o...   \n",
       "760  \\nPatient: Ms. Smith, 32 years old, G1P0, pres...   \n",
       "100  \\nPatient: [Name]\\nDate of Visit: [Date]\\nReas...   \n",
       "\n",
       "                                          conversation         model_name  \\\n",
       "93   Doctor: Hi there, my name is Dr. X. Can you te...  mistral-7b-direct   \n",
       "452  Doctor: Hi, I am Dr. X. How can I help you tod...  mistral-7b-direct   \n",
       "689  Doctor: Good morning, what brings you here tod...  mistral-7b-direct   \n",
       "302  Doctor: Good morning, what brings you to the e...  mistral-7b-direct   \n",
       "464  Doctor: Hello, how can I help you today?\\nPati...  mistral-7b-direct   \n",
       "..                                                 ...                ...   \n",
       "971  Doctor: Good morning, how are you today?\\nPati...  mistral-7b-direct   \n",
       "103  Doctor: Good morning, how are you today?\\nPati...  mistral-7b-direct   \n",
       "846  Doctor: Hello, I'm Dr. X. What brings you here...  mistral-7b-direct   \n",
       "760  Doctor: Good morning, I am Dr. Johnson. How ar...  mistral-7b-direct   \n",
       "100  Doctor: Hello, how are you today?\\nPatient: I'...  mistral-7b-direct   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "93   {'gold': 'A 37-year-old Caucasian woman was ad...    93  0.446003   \n",
       "452  {'gold': 'A female child aged 12 years, initia...   452  0.489894   \n",
       "689  {'gold': 'A 77-year-old male with a past medic...   689  0.550962   \n",
       "302  {'gold': 'A 32-year-old male presented to the ...   302  0.437850   \n",
       "464  {'gold': 'A 24-year-old female patient present...   464  0.471555   \n",
       "..                                                 ...   ...       ...   \n",
       "971  {'gold': 'A 36-year-old female, Gravida 2, Par...   971  0.000000   \n",
       "103  {'gold': 'A 74-year old male with a history of...   103  0.000000   \n",
       "846  {'gold': 'A 25-year-old Caucasian female under...   846  0.000000   \n",
       "760  {'gold': 'A 29-year-old woman, gravida 1, para...   760  0.000000   \n",
       "100  {'gold': 'A 26-year-old male patient was suffe...   100  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "93   0.841346  0.700483  0.802885   0.802885  0.918869                     8   \n",
       "452  0.837529  0.735632  0.810069   0.810069  0.899905                     2   \n",
       "689  0.808399  0.670185  0.766404   0.766404  0.867168                     8   \n",
       "302  0.817708  0.628272  0.760417   0.760417  0.919878                     8   \n",
       "464  0.794444  0.653631  0.738889   0.738889  0.915943                     7   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "971  0.009346  0.000000  0.009346   0.009346  0.797113                     1   \n",
       "103  0.019512  0.000000  0.019512   0.019512  0.656015                     2   \n",
       "846  0.017857  0.000000  0.017857   0.017857  0.799412                     1   \n",
       "760  0.017778  0.000000  0.008889   0.008889  0.809999                     1   \n",
       "100  0.019481  0.000000  0.019481   0.019481  0.688761                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "93                   7                    9                    10   \n",
       "452                  8                    9                     9   \n",
       "689                  8                    7                     6   \n",
       "302                  7                    7                    10   \n",
       "464                  9                    9                     7   \n",
       "..                 ...                  ...                   ...   \n",
       "971                  9                    9                    10   \n",
       "103                  8                    8                    10   \n",
       "846                  7                    7                     7   \n",
       "760                  5                    6                    10   \n",
       "100                  6                    4                     4   \n",
       "\n",
       "     aggregated_score  \n",
       "93           0.762323  \n",
       "452          0.754025  \n",
       "689          0.728508  \n",
       "302          0.718136  \n",
       "464          0.716090  \n",
       "..                ...  \n",
       "971          0.106051  \n",
       "103          0.104837  \n",
       "846          0.088427  \n",
       "760          0.083226  \n",
       "100          0.064298  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gpt3-direct\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>465</th>\n",
       "      <td>56341</td>\n",
       "      <td>A 46-year-old woman presented with intermitten...</td>\n",
       "      <td>Patient presented with a history of intermitte...</td>\n",
       "      <td>Doctor: Hello, I am Dr. Smith. How can I help ...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 46-year-old woman presented with i...</td>\n",
       "      <td>465</td>\n",
       "      <td>0.598169</td>\n",
       "      <td>0.838852</td>\n",
       "      <td>0.740576</td>\n",
       "      <td>0.825607</td>\n",
       "      <td>0.825607</td>\n",
       "      <td>0.933021</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.803182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>63984</td>\n",
       "      <td>A 16-year-old female presented on August 25, 2...</td>\n",
       "      <td>Patient presented with a chief complaint of na...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 16-year-old female presented on Au...</td>\n",
       "      <td>364</td>\n",
       "      <td>0.551883</td>\n",
       "      <td>0.859375</td>\n",
       "      <td>0.743455</td>\n",
       "      <td>0.796875</td>\n",
       "      <td>0.796875</td>\n",
       "      <td>0.920314</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.788300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>4182</td>\n",
       "      <td>A 53-year-old male was referred to the Oral Me...</td>\n",
       "      <td>Mr. X was seen in the Oral Medicine Department...</td>\n",
       "      <td>Doctor: Good morning, sir. I understand that y...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 53-year-old male was referred to t...</td>\n",
       "      <td>565</td>\n",
       "      <td>0.592184</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.712963</td>\n",
       "      <td>0.806452</td>\n",
       "      <td>0.806452</td>\n",
       "      <td>0.950346</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.781264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>625</th>\n",
       "      <td>27232</td>\n",
       "      <td>A 15-year-old male presented to the clinic wit...</td>\n",
       "      <td>Subjective: The patient presents with complain...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 15-year-old male presented to the ...</td>\n",
       "      <td>625</td>\n",
       "      <td>0.613840</td>\n",
       "      <td>0.838407</td>\n",
       "      <td>0.672941</td>\n",
       "      <td>0.786885</td>\n",
       "      <td>0.786885</td>\n",
       "      <td>0.942379</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.774065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>77341</td>\n",
       "      <td>A 59-year-old male with end-stage renal diseas...</td>\n",
       "      <td>Clinical Note:\\n\\nThis is a clinical note for ...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 59-year-old male with end-stage re...</td>\n",
       "      <td>208</td>\n",
       "      <td>0.515833</td>\n",
       "      <td>0.853211</td>\n",
       "      <td>0.700461</td>\n",
       "      <td>0.747706</td>\n",
       "      <td>0.747706</td>\n",
       "      <td>0.921454</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.758435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>691</th>\n",
       "      <td>2711</td>\n",
       "      <td>A 70-year-old man presented in 2017 after disc...</td>\n",
       "      <td>Patient is a [age] year old [gender] with a pa...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 70-year-old man presented in 2017 ...</td>\n",
       "      <td>691</td>\n",
       "      <td>0.000567</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>0.112903</td>\n",
       "      <td>0.184000</td>\n",
       "      <td>0.184000</td>\n",
       "      <td>0.814555</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.212700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311</th>\n",
       "      <td>1380</td>\n",
       "      <td>Our patient, an eight-year-old girl, presented...</td>\n",
       "      <td>Patient presented with a complaint of a swelli...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'Our patient, an eight-year-old girl,...</td>\n",
       "      <td>311</td>\n",
       "      <td>0.007705</td>\n",
       "      <td>0.157025</td>\n",
       "      <td>0.091667</td>\n",
       "      <td>0.115702</td>\n",
       "      <td>0.115702</td>\n",
       "      <td>0.779867</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.210185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>54758</td>\n",
       "      <td>A 4 year-old previously healthy girl was trans...</td>\n",
       "      <td>Patient presented to the clinic with complaint...</td>\n",
       "      <td>Doctor: Hello, I am Dr. Smith. How are you fee...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 4 year-old previously healthy girl...</td>\n",
       "      <td>245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.229249</td>\n",
       "      <td>0.023904</td>\n",
       "      <td>0.126482</td>\n",
       "      <td>0.126482</td>\n",
       "      <td>0.780612</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.184247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>412</th>\n",
       "      <td>107482</td>\n",
       "      <td>A 56-year-old woman with a history of breast c...</td>\n",
       "      <td>Clinical Note\\n\\nPatient: The patient is a 52-...</td>\n",
       "      <td>Doctor: Hello, I am Dr. X. What brings you her...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 56-year-old woman with a history o...</td>\n",
       "      <td>412</td>\n",
       "      <td>0.003428</td>\n",
       "      <td>0.143460</td>\n",
       "      <td>0.068085</td>\n",
       "      <td>0.118143</td>\n",
       "      <td>0.118143</td>\n",
       "      <td>0.746065</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.180584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>637</th>\n",
       "      <td>30370</td>\n",
       "      <td>A 54-year-old man diagnosed with locally advan...</td>\n",
       "      <td>Patient: The patient is a 63-year-old male who...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>{'gold': 'A 54-year-old man diagnosed with loc...</td>\n",
       "      <td>637</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.063492</td>\n",
       "      <td>0.021390</td>\n",
       "      <td>0.063492</td>\n",
       "      <td>0.063492</td>\n",
       "      <td>0.775037</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.133327</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "465   56341  A 46-year-old woman presented with intermitten...   \n",
       "364   63984  A 16-year-old female presented on August 25, 2...   \n",
       "565    4182  A 53-year-old male was referred to the Oral Me...   \n",
       "625   27232  A 15-year-old male presented to the clinic wit...   \n",
       "208   77341  A 59-year-old male with end-stage renal diseas...   \n",
       "..      ...                                                ...   \n",
       "691    2711  A 70-year-old man presented in 2017 after disc...   \n",
       "311    1380  Our patient, an eight-year-old girl, presented...   \n",
       "245   54758  A 4 year-old previously healthy girl was trans...   \n",
       "412  107482  A 56-year-old woman with a history of breast c...   \n",
       "637   30370  A 54-year-old man diagnosed with locally advan...   \n",
       "\n",
       "                                                  pred  \\\n",
       "465  Patient presented with a history of intermitte...   \n",
       "364  Patient presented with a chief complaint of na...   \n",
       "565  Mr. X was seen in the Oral Medicine Department...   \n",
       "625  Subjective: The patient presents with complain...   \n",
       "208  Clinical Note:\\n\\nThis is a clinical note for ...   \n",
       "..                                                 ...   \n",
       "691  Patient is a [age] year old [gender] with a pa...   \n",
       "311  Patient presented with a complaint of a swelli...   \n",
       "245  Patient presented to the clinic with complaint...   \n",
       "412  Clinical Note\\n\\nPatient: The patient is a 52-...   \n",
       "637  Patient: The patient is a 63-year-old male who...   \n",
       "\n",
       "                                          conversation   model_name  \\\n",
       "465  Doctor: Hello, I am Dr. Smith. How can I help ...  gpt3-direct   \n",
       "364  Doctor: Hi, how are you feeling today?\\nPatien...  gpt3-direct   \n",
       "565  Doctor: Good morning, sir. I understand that y...  gpt3-direct   \n",
       "625  Doctor: Good morning, what brings you here tod...  gpt3-direct   \n",
       "208  Doctor: Good morning, how are you feeling toda...  gpt3-direct   \n",
       "..                                                 ...          ...   \n",
       "691  Doctor: Hi, how are you feeling today?\\nPatien...  gpt3-direct   \n",
       "311  Doctor: Hello, how are you feeling today?\\nPat...  gpt3-direct   \n",
       "245  Doctor: Hello, I am Dr. Smith. How are you fee...  gpt3-direct   \n",
       "412  Doctor: Hello, I am Dr. X. What brings you her...  gpt3-direct   \n",
       "637  Doctor: Hello, how are you feeling today?\\nPat...  gpt3-direct   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "465  {'gold': 'A 46-year-old woman presented with i...   465  0.598169   \n",
       "364  {'gold': 'A 16-year-old female presented on Au...   364  0.551883   \n",
       "565  {'gold': 'A 53-year-old male was referred to t...   565  0.592184   \n",
       "625  {'gold': 'A 15-year-old male presented to the ...   625  0.613840   \n",
       "208  {'gold': 'A 59-year-old male with end-stage re...   208  0.515833   \n",
       "..                                                 ...   ...       ...   \n",
       "691  {'gold': 'A 70-year-old man presented in 2017 ...   691  0.000567   \n",
       "311  {'gold': 'Our patient, an eight-year-old girl,...   311  0.007705   \n",
       "245  {'gold': 'A 4 year-old previously healthy girl...   245  0.000000   \n",
       "412  {'gold': 'A 56-year-old woman with a history o...   412  0.003428   \n",
       "637  {'gold': 'A 54-year-old man diagnosed with loc...   637  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "465  0.838852  0.740576  0.825607   0.825607  0.933021                     9   \n",
       "364  0.859375  0.743455  0.796875   0.796875  0.920314                     9   \n",
       "565  0.857143  0.712963  0.806452   0.806452  0.950346                     7   \n",
       "625  0.838407  0.672941  0.786885   0.786885  0.942379                     9   \n",
       "208  0.853211  0.700461  0.747706   0.747706  0.921454                     8   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "691  0.200000  0.112903  0.184000   0.184000  0.814555                     1   \n",
       "311  0.157025  0.091667  0.115702   0.115702  0.779867                     7   \n",
       "245  0.229249  0.023904  0.126482   0.126482  0.780612                     3   \n",
       "412  0.143460  0.068085  0.118143   0.118143  0.746065                     2   \n",
       "637  0.063492  0.021390  0.063492   0.063492  0.775037                     2   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "465                  8                   10                    10   \n",
       "364                  9                    8                    10   \n",
       "565                  8                    9                    10   \n",
       "625                  9                    9                    10   \n",
       "208                  9                   10                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "691                  6                    7                    10   \n",
       "311                  8                    9                    10   \n",
       "245                  7                    7                    10   \n",
       "412                  8                    8                    10   \n",
       "637                  8                    7                    10   \n",
       "\n",
       "     aggregated_score  \n",
       "465          0.803182  \n",
       "364          0.788300  \n",
       "565          0.781264  \n",
       "625          0.774065  \n",
       "208          0.758435  \n",
       "..                ...  \n",
       "691          0.212700  \n",
       "311          0.210185  \n",
       "245          0.184247  \n",
       "412          0.180584  \n",
       "637          0.133327  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gpt4-direct\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>466</th>\n",
       "      <td>117607</td>\n",
       "      <td>The author reports a case of 30-year-old male ...</td>\n",
       "      <td>Patient: Male\\nAge: Not provided\\nChief compla...</td>\n",
       "      <td>Doctor: Hello, how are you today?\\nPatient: I'...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'The author reports a case of 30-year...</td>\n",
       "      <td>466</td>\n",
       "      <td>0.458397</td>\n",
       "      <td>0.761099</td>\n",
       "      <td>0.607219</td>\n",
       "      <td>0.744186</td>\n",
       "      <td>0.744186</td>\n",
       "      <td>0.918314</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.710472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>551</th>\n",
       "      <td>150217</td>\n",
       "      <td>A 38 year-old man presented to the outpatient ...</td>\n",
       "      <td>Subjective:\\nThe patient, a male, presents wit...</td>\n",
       "      <td>Doctor: Good morning, sir. How can I help you ...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 38 year-old man presented to the o...</td>\n",
       "      <td>551</td>\n",
       "      <td>0.443136</td>\n",
       "      <td>0.770270</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.698198</td>\n",
       "      <td>0.698198</td>\n",
       "      <td>0.898546</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.691200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>95126</td>\n",
       "      <td>A 62-year-old man presented with dry cough of ...</td>\n",
       "      <td>Subjective:\\nThe patient is a former smoker wi...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 62-year-old man presented with dry...</td>\n",
       "      <td>508</td>\n",
       "      <td>0.289042</td>\n",
       "      <td>0.780488</td>\n",
       "      <td>0.588235</td>\n",
       "      <td>0.726829</td>\n",
       "      <td>0.726829</td>\n",
       "      <td>0.900686</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.684849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>108552</td>\n",
       "      <td>A 43-years-old male suffered a high velocity m...</td>\n",
       "      <td>Patient: Male, [Age Not Provided]\\n\\nSubjectiv...</td>\n",
       "      <td>Doctor: Good morning, sir. How are you feeling...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 43-years-old male suffered a high ...</td>\n",
       "      <td>468</td>\n",
       "      <td>0.436965</td>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.544643</td>\n",
       "      <td>0.644444</td>\n",
       "      <td>0.644444</td>\n",
       "      <td>0.894762</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.653244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>279</th>\n",
       "      <td>202620</td>\n",
       "      <td>A 32y old woman was referred to Yas hospital d...</td>\n",
       "      <td>Patient Name: [Redacted]\\nDate: [Note: The dat...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 32y old woman was referred to Yas ...</td>\n",
       "      <td>279</td>\n",
       "      <td>0.342318</td>\n",
       "      <td>0.738776</td>\n",
       "      <td>0.561475</td>\n",
       "      <td>0.644898</td>\n",
       "      <td>0.644898</td>\n",
       "      <td>0.899342</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.647859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>182292</td>\n",
       "      <td>A 13-year-old Caucasian female was referred by...</td>\n",
       "      <td>Patient Name: Sarah\\nDate: [Date of Appointmen...</td>\n",
       "      <td>Doctor: Hello, I'm Dr. Smith. You've been refe...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 13-year-old Caucasian female was r...</td>\n",
       "      <td>286</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.117021</td>\n",
       "      <td>0.010695</td>\n",
       "      <td>0.037234</td>\n",
       "      <td>0.037234</td>\n",
       "      <td>0.752647</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.128549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>916</th>\n",
       "      <td>121545</td>\n",
       "      <td>An 8-month-old girl presented in the pediatric...</td>\n",
       "      <td>Patient: 8-month-old female\\n\\nChief Complaint...</td>\n",
       "      <td>Doctor: Hi, what brings you here today?\\nPatie...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'An 8-month-old girl presented in the...</td>\n",
       "      <td>916</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.063830</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042553</td>\n",
       "      <td>0.042553</td>\n",
       "      <td>0.748071</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.120024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>11086</td>\n",
       "      <td>This female patient in her 30s with no previou...</td>\n",
       "      <td>Patient: [Name: X] [Gender: X] [Age: X]\\n\\nSub...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'This female patient in her 30s with ...</td>\n",
       "      <td>572</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.063415</td>\n",
       "      <td>0.004902</td>\n",
       "      <td>0.034146</td>\n",
       "      <td>0.034146</td>\n",
       "      <td>0.738664</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.114011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>690</th>\n",
       "      <td>35251</td>\n",
       "      <td>A 26-year-old male presented with persistent p...</td>\n",
       "      <td>Patient: [Name: Sarah (hypothetical name used ...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 26-year-old male presented with pe...</td>\n",
       "      <td>690</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.137363</td>\n",
       "      <td>0.005525</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.757844</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.110898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>16693</td>\n",
       "      <td>A 65-year-old lady with a past medical history...</td>\n",
       "      <td>Patient: [Name Redacted]\\nAge: [Redacted]\\nGen...</td>\n",
       "      <td>Doctor: Good morning, how can I help you today...</td>\n",
       "      <td>gpt4-direct</td>\n",
       "      <td>{'gold': 'A 65-year-old lady with a past medic...</td>\n",
       "      <td>738</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.108747</td>\n",
       "      <td>0.014252</td>\n",
       "      <td>0.056738</td>\n",
       "      <td>0.056738</td>\n",
       "      <td>0.755241</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0.102984</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "466  117607  The author reports a case of 30-year-old male ...   \n",
       "551  150217  A 38 year-old man presented to the outpatient ...   \n",
       "508   95126  A 62-year-old man presented with dry cough of ...   \n",
       "468  108552  A 43-years-old male suffered a high velocity m...   \n",
       "279  202620  A 32y old woman was referred to Yas hospital d...   \n",
       "..      ...                                                ...   \n",
       "286  182292  A 13-year-old Caucasian female was referred by...   \n",
       "916  121545  An 8-month-old girl presented in the pediatric...   \n",
       "572   11086  This female patient in her 30s with no previou...   \n",
       "690   35251  A 26-year-old male presented with persistent p...   \n",
       "738   16693  A 65-year-old lady with a past medical history...   \n",
       "\n",
       "                                                  pred  \\\n",
       "466  Patient: Male\\nAge: Not provided\\nChief compla...   \n",
       "551  Subjective:\\nThe patient, a male, presents wit...   \n",
       "508  Subjective:\\nThe patient is a former smoker wi...   \n",
       "468  Patient: Male, [Age Not Provided]\\n\\nSubjectiv...   \n",
       "279  Patient Name: [Redacted]\\nDate: [Note: The dat...   \n",
       "..                                                 ...   \n",
       "286  Patient Name: Sarah\\nDate: [Date of Appointmen...   \n",
       "916  Patient: 8-month-old female\\n\\nChief Complaint...   \n",
       "572  Patient: [Name: X] [Gender: X] [Age: X]\\n\\nSub...   \n",
       "690  Patient: [Name: Sarah (hypothetical name used ...   \n",
       "738  Patient: [Name Redacted]\\nAge: [Redacted]\\nGen...   \n",
       "\n",
       "                                          conversation   model_name  \\\n",
       "466  Doctor: Hello, how are you today?\\nPatient: I'...  gpt4-direct   \n",
       "551  Doctor: Good morning, sir. How can I help you ...  gpt4-direct   \n",
       "508  Doctor: Hello, how are you feeling today?\\nPat...  gpt4-direct   \n",
       "468  Doctor: Good morning, sir. How are you feeling...  gpt4-direct   \n",
       "279  Doctor: Hello, how are you feeling today?\\nPat...  gpt4-direct   \n",
       "..                                                 ...          ...   \n",
       "286  Doctor: Hello, I'm Dr. Smith. You've been refe...  gpt4-direct   \n",
       "916  Doctor: Hi, what brings you here today?\\nPatie...  gpt4-direct   \n",
       "572  Doctor: Hi, how are you feeling today?\\nPatien...  gpt4-direct   \n",
       "690  Doctor: Good morning, what brings you here tod...  gpt4-direct   \n",
       "738  Doctor: Good morning, how can I help you today...  gpt4-direct   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "466  {'gold': 'The author reports a case of 30-year...   466  0.458397   \n",
       "551  {'gold': 'A 38 year-old man presented to the o...   551  0.443136   \n",
       "508  {'gold': 'A 62-year-old man presented with dry...   508  0.289042   \n",
       "468  {'gold': 'A 43-years-old male suffered a high ...   468  0.436965   \n",
       "279  {'gold': 'A 32y old woman was referred to Yas ...   279  0.342318   \n",
       "..                                                 ...   ...       ...   \n",
       "286  {'gold': 'A 13-year-old Caucasian female was r...   286  0.000000   \n",
       "916  {'gold': 'An 8-month-old girl presented in the...   916  0.000000   \n",
       "572  {'gold': 'This female patient in her 30s with ...   572  0.000000   \n",
       "690  {'gold': 'A 26-year-old male presented with pe...   690  0.000000   \n",
       "738  {'gold': 'A 65-year-old lady with a past medic...   738  0.000000   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "466  0.761099  0.607219  0.744186   0.744186  0.918314                     9   \n",
       "551  0.770270  0.588235  0.698198   0.698198  0.898546                     9   \n",
       "508  0.780488  0.588235  0.726829   0.726829  0.900686                     9   \n",
       "468  0.706667  0.544643  0.644444   0.644444  0.894762                     9   \n",
       "279  0.738776  0.561475  0.644898   0.644898  0.899342                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "286  0.117021  0.010695  0.037234   0.037234  0.752647                     9   \n",
       "916  0.063830  0.000000  0.042553   0.042553  0.748071                     9   \n",
       "572  0.063415  0.004902  0.034146   0.034146  0.738664                     9   \n",
       "690  0.137363  0.005525  0.071429   0.071429  0.757844                     1   \n",
       "738  0.108747  0.014252  0.056738   0.056738  0.755241                     2   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "466                  7                    9                    10   \n",
       "551                  8                    9                    10   \n",
       "508                  7                    8                    10   \n",
       "468                  8                   10                    10   \n",
       "279                  8                    8                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "286                  8                    7                     1   \n",
       "916                  8                    7                     1   \n",
       "572                  7                    7                     1   \n",
       "690                  7                    7                     1   \n",
       "738                  7                    6                     1   \n",
       "\n",
       "     aggregated_score  \n",
       "466          0.710472  \n",
       "551          0.691200  \n",
       "508          0.684849  \n",
       "468          0.653244  \n",
       "279          0.647859  \n",
       "..                ...  \n",
       "286          0.128549  \n",
       "916          0.120024  \n",
       "572          0.114011  \n",
       "690          0.110898  \n",
       "738          0.102984  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gpt3-generator-gpt\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>552</th>\n",
       "      <td>4615</td>\n",
       "      <td>A 30-year-old female presented with a 5-week h...</td>\n",
       "      <td>Patient presents with a painful rapidly-growin...</td>\n",
       "      <td>Doctor: Good morning, how can I help you today...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 30-year-old female presented with ...</td>\n",
       "      <td>552</td>\n",
       "      <td>0.548937</td>\n",
       "      <td>0.837321</td>\n",
       "      <td>0.721154</td>\n",
       "      <td>0.794258</td>\n",
       "      <td>0.794258</td>\n",
       "      <td>0.909886</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.778242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>157147</td>\n",
       "      <td>A 65-year-old man presented to the emergency d...</td>\n",
       "      <td>Clinical Note:\\n\\nThis is a clinical note for ...</td>\n",
       "      <td>Doctor: Good morning, can you tell me what hap...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 65-year-old man presented to the e...</td>\n",
       "      <td>781</td>\n",
       "      <td>0.525842</td>\n",
       "      <td>0.784119</td>\n",
       "      <td>0.648379</td>\n",
       "      <td>0.694789</td>\n",
       "      <td>0.694789</td>\n",
       "      <td>0.921207</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0.699674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>968</th>\n",
       "      <td>73125</td>\n",
       "      <td>A 79-year-old Caucasian female presented to ou...</td>\n",
       "      <td>Patient presented with a chief complaint of ch...</td>\n",
       "      <td>Doctor: Good morning, Mrs. How can I help you ...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 79-year-old Caucasian female prese...</td>\n",
       "      <td>968</td>\n",
       "      <td>0.371155</td>\n",
       "      <td>0.744048</td>\n",
       "      <td>0.598802</td>\n",
       "      <td>0.696429</td>\n",
       "      <td>0.696429</td>\n",
       "      <td>0.905819</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.686856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>305</th>\n",
       "      <td>96959</td>\n",
       "      <td>A 26-year-old man from Albania presented with ...</td>\n",
       "      <td>Clinical Note:\\n\\nPatient Summary:\\nA 26-year-...</td>\n",
       "      <td>Doctor: Good morning, how are you today?\\nPati...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 26-year-old man from Albania prese...</td>\n",
       "      <td>305</td>\n",
       "      <td>0.444729</td>\n",
       "      <td>0.771242</td>\n",
       "      <td>0.568928</td>\n",
       "      <td>0.631808</td>\n",
       "      <td>0.631808</td>\n",
       "      <td>0.917579</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.661490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>703</th>\n",
       "      <td>13468</td>\n",
       "      <td>A 14-year-old girl without any previous clinic...</td>\n",
       "      <td>Patient, a 14-year-old female student, present...</td>\n",
       "      <td>Doctor: Hi, my name is Dr. Smith. Can you tell...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 14-year-old girl without any previ...</td>\n",
       "      <td>703</td>\n",
       "      <td>0.443759</td>\n",
       "      <td>0.756614</td>\n",
       "      <td>0.585106</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>0.619048</td>\n",
       "      <td>0.907943</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.660995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>676</th>\n",
       "      <td>34663</td>\n",
       "      <td>A 62 year old woman was diagnosed with B-cell ...</td>\n",
       "      <td>This is a clinical note for a 62-year-old woma...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 62 year old woman was diagnosed wi...</td>\n",
       "      <td>676</td>\n",
       "      <td>0.048701</td>\n",
       "      <td>0.305835</td>\n",
       "      <td>0.141414</td>\n",
       "      <td>0.201207</td>\n",
       "      <td>0.201207</td>\n",
       "      <td>0.816796</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.271800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>86578</td>\n",
       "      <td>A 34-year-old female was seen in our outpatien...</td>\n",
       "      <td>Subjective:\\nThe patient, a 34-year-old female...</td>\n",
       "      <td>Doctor: Good morning, what brings you to our c...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 34-year-old female was seen in our...</td>\n",
       "      <td>273</td>\n",
       "      <td>0.100322</td>\n",
       "      <td>0.283619</td>\n",
       "      <td>0.157248</td>\n",
       "      <td>0.156479</td>\n",
       "      <td>0.156479</td>\n",
       "      <td>0.803859</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.268724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>948</th>\n",
       "      <td>62887</td>\n",
       "      <td>A 75-year-old Caucasian male presented to the ...</td>\n",
       "      <td>Patient: Male, 75 years old, of Caucasian ethn...</td>\n",
       "      <td>Doctor: Good morning, sir. I'm Dr. Smith. What...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 75-year-old Caucasian male present...</td>\n",
       "      <td>948</td>\n",
       "      <td>0.079810</td>\n",
       "      <td>0.306748</td>\n",
       "      <td>0.123457</td>\n",
       "      <td>0.184049</td>\n",
       "      <td>0.184049</td>\n",
       "      <td>0.792763</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0.263757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>557</th>\n",
       "      <td>181391</td>\n",
       "      <td>A 70-year-old Sri Lankan Tamil male with a his...</td>\n",
       "      <td>The patient is a 70-year-old male of Sri Lanka...</td>\n",
       "      <td>Doctor: Good morning, Mr. Jones. I've reviewed...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A 70-year-old Sri Lankan Tamil male ...</td>\n",
       "      <td>557</td>\n",
       "      <td>0.060501</td>\n",
       "      <td>0.218289</td>\n",
       "      <td>0.100890</td>\n",
       "      <td>0.117994</td>\n",
       "      <td>0.117994</td>\n",
       "      <td>0.782342</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.212336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>92280</td>\n",
       "      <td>A nine-year-old boy presented to our outpatien...</td>\n",
       "      <td>Clinical Note:\\n\\nThis is a nine-year-old male...</td>\n",
       "      <td>Doctor: Hello, what brings you here today?\\nPa...</td>\n",
       "      <td>gpt3-generator-gpt</td>\n",
       "      <td>{'gold': 'A nine-year-old boy presented to our...</td>\n",
       "      <td>10</td>\n",
       "      <td>0.038661</td>\n",
       "      <td>0.170213</td>\n",
       "      <td>0.085714</td>\n",
       "      <td>0.099291</td>\n",
       "      <td>0.099291</td>\n",
       "      <td>0.775800</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>4</td>\n",
       "      <td>0.186216</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "552    4615  A 30-year-old female presented with a 5-week h...   \n",
       "781  157147  A 65-year-old man presented to the emergency d...   \n",
       "968   73125  A 79-year-old Caucasian female presented to ou...   \n",
       "305   96959  A 26-year-old man from Albania presented with ...   \n",
       "703   13468  A 14-year-old girl without any previous clinic...   \n",
       "..      ...                                                ...   \n",
       "676   34663  A 62 year old woman was diagnosed with B-cell ...   \n",
       "273   86578  A 34-year-old female was seen in our outpatien...   \n",
       "948   62887  A 75-year-old Caucasian male presented to the ...   \n",
       "557  181391  A 70-year-old Sri Lankan Tamil male with a his...   \n",
       "10    92280  A nine-year-old boy presented to our outpatien...   \n",
       "\n",
       "                                                  pred  \\\n",
       "552  Patient presents with a painful rapidly-growin...   \n",
       "781  Clinical Note:\\n\\nThis is a clinical note for ...   \n",
       "968  Patient presented with a chief complaint of ch...   \n",
       "305  Clinical Note:\\n\\nPatient Summary:\\nA 26-year-...   \n",
       "703  Patient, a 14-year-old female student, present...   \n",
       "..                                                 ...   \n",
       "676  This is a clinical note for a 62-year-old woma...   \n",
       "273  Subjective:\\nThe patient, a 34-year-old female...   \n",
       "948  Patient: Male, 75 years old, of Caucasian ethn...   \n",
       "557  The patient is a 70-year-old male of Sri Lanka...   \n",
       "10   Clinical Note:\\n\\nThis is a nine-year-old male...   \n",
       "\n",
       "                                          conversation          model_name  \\\n",
       "552  Doctor: Good morning, how can I help you today...  gpt3-generator-gpt   \n",
       "781  Doctor: Good morning, can you tell me what hap...  gpt3-generator-gpt   \n",
       "968  Doctor: Good morning, Mrs. How can I help you ...  gpt3-generator-gpt   \n",
       "305  Doctor: Good morning, how are you today?\\nPati...  gpt3-generator-gpt   \n",
       "703  Doctor: Hi, my name is Dr. Smith. Can you tell...  gpt3-generator-gpt   \n",
       "..                                                 ...                 ...   \n",
       "676  Doctor: Hello, how are you feeling today?\\nPat...  gpt3-generator-gpt   \n",
       "273  Doctor: Good morning, what brings you to our c...  gpt3-generator-gpt   \n",
       "948  Doctor: Good morning, sir. I'm Dr. Smith. What...  gpt3-generator-gpt   \n",
       "557  Doctor: Good morning, Mr. Jones. I've reviewed...  gpt3-generator-gpt   \n",
       "10   Doctor: Hello, what brings you here today?\\nPa...  gpt3-generator-gpt   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "552  {'gold': 'A 30-year-old female presented with ...   552  0.548937   \n",
       "781  {'gold': 'A 65-year-old man presented to the e...   781  0.525842   \n",
       "968  {'gold': 'A 79-year-old Caucasian female prese...   968  0.371155   \n",
       "305  {'gold': 'A 26-year-old man from Albania prese...   305  0.444729   \n",
       "703  {'gold': 'A 14-year-old girl without any previ...   703  0.443759   \n",
       "..                                                 ...   ...       ...   \n",
       "676  {'gold': 'A 62 year old woman was diagnosed wi...   676  0.048701   \n",
       "273  {'gold': 'A 34-year-old female was seen in our...   273  0.100322   \n",
       "948  {'gold': 'A 75-year-old Caucasian male present...   948  0.079810   \n",
       "557  {'gold': 'A 70-year-old Sri Lankan Tamil male ...   557  0.060501   \n",
       "10   {'gold': 'A nine-year-old boy presented to our...    10  0.038661   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "552  0.837321  0.721154  0.794258   0.794258  0.909886                     9   \n",
       "781  0.784119  0.648379  0.694789   0.694789  0.921207                     9   \n",
       "968  0.744048  0.598802  0.696429   0.696429  0.905819                     9   \n",
       "305  0.771242  0.568928  0.631808   0.631808  0.917579                     9   \n",
       "703  0.756614  0.585106  0.619048   0.619048  0.907943                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "676  0.305835  0.141414  0.201207   0.201207  0.816796                     8   \n",
       "273  0.283619  0.157248  0.156479   0.156479  0.803859                     8   \n",
       "948  0.306748  0.123457  0.184049   0.184049  0.792763                     8   \n",
       "557  0.218289  0.100890  0.117994   0.117994  0.782342                     5   \n",
       "10   0.170213  0.085714  0.099291   0.099291  0.775800                     7   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "552                  8                    9                    10   \n",
       "781                  7                    7                     9   \n",
       "968                  9                    9                    10   \n",
       "305                  9                    8                    10   \n",
       "703                  9                    9                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "676                  5                    8                    10   \n",
       "273                  8                    8                    10   \n",
       "948                  7                    7                     9   \n",
       "557                  6                    9                    10   \n",
       "10                   7                    9                     4   \n",
       "\n",
       "     aggregated_score  \n",
       "552          0.778242  \n",
       "781          0.699674  \n",
       "968          0.686856  \n",
       "305          0.661490  \n",
       "703          0.660995  \n",
       "..                ...  \n",
       "676          0.271800  \n",
       "273          0.268724  \n",
       "948          0.263757  \n",
       "557          0.212336  \n",
       "10           0.186216  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gpt3-generator-7b\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>969</th>\n",
       "      <td>81559</td>\n",
       "      <td>The patient is a 36-year-old male with bicuspi...</td>\n",
       "      <td>The patient, a 36-year-old male, presented to ...</td>\n",
       "      <td>Doctor: Good morning, sir. Can you tell me you...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'The patient is a 36-year-old male wi...</td>\n",
       "      <td>969</td>\n",
       "      <td>0.533346</td>\n",
       "      <td>0.810934</td>\n",
       "      <td>0.691076</td>\n",
       "      <td>0.701595</td>\n",
       "      <td>0.701595</td>\n",
       "      <td>0.906316</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>0.727352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>543</th>\n",
       "      <td>99849</td>\n",
       "      <td>A 56-year-old male with history of alcoholic l...</td>\n",
       "      <td>Patient Summary:\\n\\nThe patient is a 56-year-o...</td>\n",
       "      <td>Doctor: Good morning, sir. I understand you ha...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 56-year-old male with history of a...</td>\n",
       "      <td>543</td>\n",
       "      <td>0.482289</td>\n",
       "      <td>0.784483</td>\n",
       "      <td>0.593074</td>\n",
       "      <td>0.711207</td>\n",
       "      <td>0.711207</td>\n",
       "      <td>0.922035</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.703316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>660</th>\n",
       "      <td>92906</td>\n",
       "      <td>A 54-year-old female complained of a slowly gr...</td>\n",
       "      <td>A 54-year-old female presented with a complain...</td>\n",
       "      <td>1. Doctor: Good morning, what brings you here ...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 54-year-old female complained of a...</td>\n",
       "      <td>660</td>\n",
       "      <td>0.542131</td>\n",
       "      <td>0.788462</td>\n",
       "      <td>0.647343</td>\n",
       "      <td>0.634615</td>\n",
       "      <td>0.634615</td>\n",
       "      <td>0.930085</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.687640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>183055</td>\n",
       "      <td>We report a male baby born in a peripheral set...</td>\n",
       "      <td>Patient: \\n\\nThe patient is a newborn male who...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'We report a male baby born in a peri...</td>\n",
       "      <td>890</td>\n",
       "      <td>0.576397</td>\n",
       "      <td>0.824121</td>\n",
       "      <td>0.661616</td>\n",
       "      <td>0.678392</td>\n",
       "      <td>0.678392</td>\n",
       "      <td>0.898149</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>0.686280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>60564</td>\n",
       "      <td>A 29-year-old lady presented to the surgical o...</td>\n",
       "      <td>Clinical Note:\\n\\nThis is a 29-year-old female...</td>\n",
       "      <td>Doctor: Good morning, what brings you to the s...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 29-year-old lady presented to the ...</td>\n",
       "      <td>179</td>\n",
       "      <td>0.497164</td>\n",
       "      <td>0.762332</td>\n",
       "      <td>0.612613</td>\n",
       "      <td>0.636771</td>\n",
       "      <td>0.636771</td>\n",
       "      <td>0.925585</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>0.674158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>111710</td>\n",
       "      <td>A 69-year-old female reported to the departmen...</td>\n",
       "      <td>Patient Information:\\n- Age: 69\\n- Sex: Female...</td>\n",
       "      <td>Doctor: Good morning, I'm Dr. Smith. What brin...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 69-year-old female reported to the...</td>\n",
       "      <td>172</td>\n",
       "      <td>0.088734</td>\n",
       "      <td>0.344423</td>\n",
       "      <td>0.196464</td>\n",
       "      <td>0.227006</td>\n",
       "      <td>0.227006</td>\n",
       "      <td>0.833320</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0.279590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>913</th>\n",
       "      <td>165259</td>\n",
       "      <td>An 8-year-old girl, studying in second grade w...</td>\n",
       "      <td>Clinical Note:\\nThis is an 8-year-old female p...</td>\n",
       "      <td>Doctor: Hello, I am Dr. Smith. How can I help ...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'An 8-year-old girl, studying in seco...</td>\n",
       "      <td>913</td>\n",
       "      <td>0.097472</td>\n",
       "      <td>0.260664</td>\n",
       "      <td>0.138095</td>\n",
       "      <td>0.175355</td>\n",
       "      <td>0.175355</td>\n",
       "      <td>0.793920</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.271559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>186368</td>\n",
       "      <td>A 14-year-old male patient presented to the ou...</td>\n",
       "      <td>Subjective:\\nThe patient, a 14-year-old male, ...</td>\n",
       "      <td>Doctor: Hello, how can I help you today?\\nPati...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 14-year-old male patient presented...</td>\n",
       "      <td>392</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.306306</td>\n",
       "      <td>0.096677</td>\n",
       "      <td>0.192192</td>\n",
       "      <td>0.192192</td>\n",
       "      <td>0.793485</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.249299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>710</th>\n",
       "      <td>6721</td>\n",
       "      <td>A 76-year-old female patient was admitted to o...</td>\n",
       "      <td>This is a clinical note for a 76-year-old fema...</td>\n",
       "      <td>Doctor: Good morning, I am Dr. X. How are you ...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'A 76-year-old female patient was adm...</td>\n",
       "      <td>710</td>\n",
       "      <td>0.078897</td>\n",
       "      <td>0.211382</td>\n",
       "      <td>0.103542</td>\n",
       "      <td>0.119241</td>\n",
       "      <td>0.119241</td>\n",
       "      <td>0.776798</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.221241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>114112</td>\n",
       "      <td>An 84-year-old female presented with progressi...</td>\n",
       "      <td>Patient Summary:\\nThis is an 84-year-old femal...</td>\n",
       "      <td>Doctor: Hi, I am Dr. X. How can I help you tod...</td>\n",
       "      <td>gpt3-generator-7b</td>\n",
       "      <td>{'gold': 'An 84-year-old female presented with...</td>\n",
       "      <td>506</td>\n",
       "      <td>0.049633</td>\n",
       "      <td>0.177778</td>\n",
       "      <td>0.104478</td>\n",
       "      <td>0.103704</td>\n",
       "      <td>0.103704</td>\n",
       "      <td>0.774906</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.196004</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "969   81559  The patient is a 36-year-old male with bicuspi...   \n",
       "543   99849  A 56-year-old male with history of alcoholic l...   \n",
       "660   92906  A 54-year-old female complained of a slowly gr...   \n",
       "890  183055  We report a male baby born in a peripheral set...   \n",
       "179   60564  A 29-year-old lady presented to the surgical o...   \n",
       "..      ...                                                ...   \n",
       "172  111710  A 69-year-old female reported to the departmen...   \n",
       "913  165259  An 8-year-old girl, studying in second grade w...   \n",
       "392  186368  A 14-year-old male patient presented to the ou...   \n",
       "710    6721  A 76-year-old female patient was admitted to o...   \n",
       "506  114112  An 84-year-old female presented with progressi...   \n",
       "\n",
       "                                                  pred  \\\n",
       "969  The patient, a 36-year-old male, presented to ...   \n",
       "543  Patient Summary:\\n\\nThe patient is a 56-year-o...   \n",
       "660  A 54-year-old female presented with a complain...   \n",
       "890  Patient: \\n\\nThe patient is a newborn male who...   \n",
       "179  Clinical Note:\\n\\nThis is a 29-year-old female...   \n",
       "..                                                 ...   \n",
       "172  Patient Information:\\n- Age: 69\\n- Sex: Female...   \n",
       "913  Clinical Note:\\nThis is an 8-year-old female p...   \n",
       "392  Subjective:\\nThe patient, a 14-year-old male, ...   \n",
       "710  This is a clinical note for a 76-year-old fema...   \n",
       "506  Patient Summary:\\nThis is an 84-year-old femal...   \n",
       "\n",
       "                                          conversation         model_name  \\\n",
       "969  Doctor: Good morning, sir. Can you tell me you...  gpt3-generator-7b   \n",
       "543  Doctor: Good morning, sir. I understand you ha...  gpt3-generator-7b   \n",
       "660  1. Doctor: Good morning, what brings you here ...  gpt3-generator-7b   \n",
       "890  Doctor: Good morning, how are you feeling toda...  gpt3-generator-7b   \n",
       "179  Doctor: Good morning, what brings you to the s...  gpt3-generator-7b   \n",
       "..                                                 ...                ...   \n",
       "172  Doctor: Good morning, I'm Dr. Smith. What brin...  gpt3-generator-7b   \n",
       "913  Doctor: Hello, I am Dr. Smith. How can I help ...  gpt3-generator-7b   \n",
       "392  Doctor: Hello, how can I help you today?\\nPati...  gpt3-generator-7b   \n",
       "710  Doctor: Good morning, I am Dr. X. How are you ...  gpt3-generator-7b   \n",
       "506  Doctor: Hi, I am Dr. X. How can I help you tod...  gpt3-generator-7b   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "969  {'gold': 'The patient is a 36-year-old male wi...   969  0.533346   \n",
       "543  {'gold': 'A 56-year-old male with history of a...   543  0.482289   \n",
       "660  {'gold': 'A 54-year-old female complained of a...   660  0.542131   \n",
       "890  {'gold': 'We report a male baby born in a peri...   890  0.576397   \n",
       "179  {'gold': 'A 29-year-old lady presented to the ...   179  0.497164   \n",
       "..                                                 ...   ...       ...   \n",
       "172  {'gold': 'A 69-year-old female reported to the...   172  0.088734   \n",
       "913  {'gold': 'An 8-year-old girl, studying in seco...   913  0.097472   \n",
       "392  {'gold': 'A 14-year-old male patient presented...   392  0.000000   \n",
       "710  {'gold': 'A 76-year-old female patient was adm...   710  0.078897   \n",
       "506  {'gold': 'An 84-year-old female presented with...   506  0.049633   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "969  0.810934  0.691076  0.701595   0.701595  0.906316                     9   \n",
       "543  0.784483  0.593074  0.711207   0.711207  0.922035                     9   \n",
       "660  0.788462  0.647343  0.634615   0.634615  0.930085                     9   \n",
       "890  0.824121  0.661616  0.678392   0.678392  0.898149                     8   \n",
       "179  0.762332  0.612613  0.636771   0.636771  0.925585                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "172  0.344423  0.196464  0.227006   0.227006  0.833320                     8   \n",
       "913  0.260664  0.138095  0.175355   0.175355  0.793920                     9   \n",
       "392  0.306306  0.096677  0.192192   0.192192  0.793485                     9   \n",
       "710  0.211382  0.103542  0.119241   0.119241  0.776798                     7   \n",
       "506  0.177778  0.104478  0.103704   0.103704  0.774906                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "969                  9                    9                     8   \n",
       "543                  8                    9                    10   \n",
       "660                  8                    8                    10   \n",
       "890                  8                    8                     1   \n",
       "179                  9                    9                     8   \n",
       "..                 ...                  ...                   ...   \n",
       "172                  5                    5                     5   \n",
       "913                  8                    8                    10   \n",
       "392                  9                    9                     1   \n",
       "710                  7                    8                    10   \n",
       "506                  9                    9                    10   \n",
       "\n",
       "     aggregated_score  \n",
       "969          0.727352  \n",
       "543          0.703316  \n",
       "660          0.687640  \n",
       "890          0.686280  \n",
       "179          0.674158  \n",
       "..                ...  \n",
       "172          0.279590  \n",
       "913          0.271559  \n",
       "392          0.249299  \n",
       "710          0.221241  \n",
       "506          0.196004  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "gpt3-generator-13b\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>961</th>\n",
       "      <td>36970</td>\n",
       "      <td>At the end of 2008, a 54-year-old Japanese man...</td>\n",
       "      <td>This patient, a 54-year-old Japanese male, pre...</td>\n",
       "      <td>Doctor: Good morning, Mr. Japanese man. I unde...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'At the end of 2008, a 54-year-old Ja...</td>\n",
       "      <td>961</td>\n",
       "      <td>0.526342</td>\n",
       "      <td>0.807775</td>\n",
       "      <td>0.637744</td>\n",
       "      <td>0.643629</td>\n",
       "      <td>0.643629</td>\n",
       "      <td>0.921412</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>0.690525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>196073</td>\n",
       "      <td>A 55-year-old man presented to the emergency d...</td>\n",
       "      <td>Clinical Note:\\nThis is a note for a 55-year-o...</td>\n",
       "      <td>Doctor: Good morning, how can I help you today...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 55-year-old man presented to the e...</td>\n",
       "      <td>208</td>\n",
       "      <td>0.475120</td>\n",
       "      <td>0.780600</td>\n",
       "      <td>0.603248</td>\n",
       "      <td>0.683603</td>\n",
       "      <td>0.683603</td>\n",
       "      <td>0.909571</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.689861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>742</th>\n",
       "      <td>86242</td>\n",
       "      <td>A 72-year-old man was referred to our hospital...</td>\n",
       "      <td>Patient, a 72-year-old male, was admitted for ...</td>\n",
       "      <td>Doctor: Good morning, Mr. Smith. I'm Dr. Smith...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 72-year-old man was referred to ou...</td>\n",
       "      <td>742</td>\n",
       "      <td>0.560724</td>\n",
       "      <td>0.776316</td>\n",
       "      <td>0.660793</td>\n",
       "      <td>0.592105</td>\n",
       "      <td>0.592105</td>\n",
       "      <td>0.910879</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.680004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>609</td>\n",
       "      <td>We describe the case of a 27-year-old white wo...</td>\n",
       "      <td>Clinical Note:\\n\\nPatient Summary:\\nThe patien...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'We describe the case of a 27-year-ol...</td>\n",
       "      <td>82</td>\n",
       "      <td>0.530151</td>\n",
       "      <td>0.790598</td>\n",
       "      <td>0.643777</td>\n",
       "      <td>0.658120</td>\n",
       "      <td>0.658120</td>\n",
       "      <td>0.930085</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.678128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>870</th>\n",
       "      <td>1380</td>\n",
       "      <td>Our patient, an eight-year-old girl, presented...</td>\n",
       "      <td>The patient, an eight-year-old female, present...</td>\n",
       "      <td>Doctor: Hello, how are you feeling today?\\nPat...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'Our patient, an eight-year-old girl,...</td>\n",
       "      <td>870</td>\n",
       "      <td>0.485247</td>\n",
       "      <td>0.773537</td>\n",
       "      <td>0.618926</td>\n",
       "      <td>0.681934</td>\n",
       "      <td>0.681934</td>\n",
       "      <td>0.916289</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>0.671588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>916</th>\n",
       "      <td>175367</td>\n",
       "      <td>A 77-year-old man with a past medical history ...</td>\n",
       "      <td>Chief Complaint: The patient presents with acu...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 77-year-old man with a past medica...</td>\n",
       "      <td>916</td>\n",
       "      <td>0.094471</td>\n",
       "      <td>0.383632</td>\n",
       "      <td>0.195373</td>\n",
       "      <td>0.214834</td>\n",
       "      <td>0.214834</td>\n",
       "      <td>0.812355</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>0.272564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>44850</td>\n",
       "      <td>A 36-year-old male presented with the chief co...</td>\n",
       "      <td>Subjective:\\nThe patient, a 36-year-old male, ...</td>\n",
       "      <td>Doctor: Good morning, sir. I'm Dr. Smith. How ...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 36-year-old male presented with th...</td>\n",
       "      <td>751</td>\n",
       "      <td>0.107041</td>\n",
       "      <td>0.356021</td>\n",
       "      <td>0.168421</td>\n",
       "      <td>0.157068</td>\n",
       "      <td>0.157068</td>\n",
       "      <td>0.791139</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>0.269050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>182599</td>\n",
       "      <td>The first case we present is that of a 29-year...</td>\n",
       "      <td>Patient Name: [Patient's Name]\\nAge: 29\\nSex: ...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'The first case we present is that of...</td>\n",
       "      <td>287</td>\n",
       "      <td>0.075740</td>\n",
       "      <td>0.292135</td>\n",
       "      <td>0.146893</td>\n",
       "      <td>0.224719</td>\n",
       "      <td>0.224719</td>\n",
       "      <td>0.800140</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0.266696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>732</th>\n",
       "      <td>202348</td>\n",
       "      <td>A 36-year-old Caucasian male presented to the ...</td>\n",
       "      <td>The patient, a 36-year-old Caucasian male, pre...</td>\n",
       "      <td>Doctor: Hi there, what brings you in today?\\nP...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 36-year-old Caucasian male present...</td>\n",
       "      <td>732</td>\n",
       "      <td>0.066791</td>\n",
       "      <td>0.238636</td>\n",
       "      <td>0.091429</td>\n",
       "      <td>0.119318</td>\n",
       "      <td>0.119318</td>\n",
       "      <td>0.790040</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.199767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>940</th>\n",
       "      <td>123492</td>\n",
       "      <td>A 57-year-old man was evaluated in the emergen...</td>\n",
       "      <td>Patient Name: John Doe\\nAge: 57\\nSex: Male\\nDa...</td>\n",
       "      <td>Doctor: Good morning, what brings you to the e...</td>\n",
       "      <td>gpt3-generator-13b</td>\n",
       "      <td>{'gold': 'A 57-year-old man was evaluated in t...</td>\n",
       "      <td>940</td>\n",
       "      <td>0.042907</td>\n",
       "      <td>0.188976</td>\n",
       "      <td>0.095238</td>\n",
       "      <td>0.165354</td>\n",
       "      <td>0.165354</td>\n",
       "      <td>0.778988</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>0.166318</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "961   36970  At the end of 2008, a 54-year-old Japanese man...   \n",
       "208  196073  A 55-year-old man presented to the emergency d...   \n",
       "742   86242  A 72-year-old man was referred to our hospital...   \n",
       "82      609  We describe the case of a 27-year-old white wo...   \n",
       "870    1380  Our patient, an eight-year-old girl, presented...   \n",
       "..      ...                                                ...   \n",
       "916  175367  A 77-year-old man with a past medical history ...   \n",
       "751   44850  A 36-year-old male presented with the chief co...   \n",
       "287  182599  The first case we present is that of a 29-year...   \n",
       "732  202348  A 36-year-old Caucasian male presented to the ...   \n",
       "940  123492  A 57-year-old man was evaluated in the emergen...   \n",
       "\n",
       "                                                  pred  \\\n",
       "961  This patient, a 54-year-old Japanese male, pre...   \n",
       "208  Clinical Note:\\nThis is a note for a 55-year-o...   \n",
       "742  Patient, a 72-year-old male, was admitted for ...   \n",
       "82   Clinical Note:\\n\\nPatient Summary:\\nThe patien...   \n",
       "870  The patient, an eight-year-old female, present...   \n",
       "..                                                 ...   \n",
       "916  Chief Complaint: The patient presents with acu...   \n",
       "751  Subjective:\\nThe patient, a 36-year-old male, ...   \n",
       "287  Patient Name: [Patient's Name]\\nAge: 29\\nSex: ...   \n",
       "732  The patient, a 36-year-old Caucasian male, pre...   \n",
       "940  Patient Name: John Doe\\nAge: 57\\nSex: Male\\nDa...   \n",
       "\n",
       "                                          conversation          model_name  \\\n",
       "961  Doctor: Good morning, Mr. Japanese man. I unde...  gpt3-generator-13b   \n",
       "208  Doctor: Good morning, how can I help you today...  gpt3-generator-13b   \n",
       "742  Doctor: Good morning, Mr. Smith. I'm Dr. Smith...  gpt3-generator-13b   \n",
       "82   Doctor: Hi, how are you feeling today?\\nPatien...  gpt3-generator-13b   \n",
       "870  Doctor: Hello, how are you feeling today?\\nPat...  gpt3-generator-13b   \n",
       "..                                                 ...                 ...   \n",
       "916  Doctor: Good morning, how are you feeling toda...  gpt3-generator-13b   \n",
       "751  Doctor: Good morning, sir. I'm Dr. Smith. How ...  gpt3-generator-13b   \n",
       "287  Doctor: Good morning, how are you feeling toda...  gpt3-generator-13b   \n",
       "732  Doctor: Hi there, what brings you in today?\\nP...  gpt3-generator-13b   \n",
       "940  Doctor: Good morning, what brings you to the e...  gpt3-generator-13b   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "961  {'gold': 'At the end of 2008, a 54-year-old Ja...   961  0.526342   \n",
       "208  {'gold': 'A 55-year-old man presented to the e...   208  0.475120   \n",
       "742  {'gold': 'A 72-year-old man was referred to ou...   742  0.560724   \n",
       "82   {'gold': 'We describe the case of a 27-year-ol...    82  0.530151   \n",
       "870  {'gold': 'Our patient, an eight-year-old girl,...   870  0.485247   \n",
       "..                                                 ...   ...       ...   \n",
       "916  {'gold': 'A 77-year-old man with a past medica...   916  0.094471   \n",
       "751  {'gold': 'A 36-year-old male presented with th...   751  0.107041   \n",
       "287  {'gold': 'The first case we present is that of...   287  0.075740   \n",
       "732  {'gold': 'A 36-year-old Caucasian male present...   732  0.066791   \n",
       "940  {'gold': 'A 57-year-old man was evaluated in t...   940  0.042907   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "961  0.807775  0.637744  0.643629   0.643629  0.921412                     7   \n",
       "208  0.780600  0.603248  0.683603   0.683603  0.909571                     9   \n",
       "742  0.776316  0.660793  0.592105   0.592105  0.910879                     9   \n",
       "82   0.790598  0.643777  0.658120   0.658120  0.930085                     9   \n",
       "870  0.773537  0.618926  0.681934   0.681934  0.916289                     9   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "916  0.383632  0.195373  0.214834   0.214834  0.812355                     2   \n",
       "751  0.356021  0.168421  0.157068   0.157068  0.791139                     6   \n",
       "287  0.292135  0.146893  0.224719   0.224719  0.800140                     7   \n",
       "732  0.238636  0.091429  0.119318   0.119318  0.790040                     1   \n",
       "940  0.188976  0.095238  0.165354   0.165354  0.778988                     1   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "961                  9                   10                     9   \n",
       "208                  8                    8                    10   \n",
       "742                  9                    9                    10   \n",
       "82                   5                    6                    10   \n",
       "870                  7                    7                     5   \n",
       "..                 ...                  ...                   ...   \n",
       "916                  7                    8                     4   \n",
       "751                  8                    7                     9   \n",
       "287                  8                    9                     1   \n",
       "732                  7                    8                    10   \n",
       "940                  5                    5                     1   \n",
       "\n",
       "     aggregated_score  \n",
       "961          0.690525  \n",
       "208          0.689861  \n",
       "742          0.680004  \n",
       "82           0.678128  \n",
       "870          0.671588  \n",
       "..                ...  \n",
       "916          0.272564  \n",
       "751          0.269050  \n",
       "287          0.266696  \n",
       "732          0.199767  \n",
       "940          0.166318  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-7b-generator-gold\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>170531</td>\n",
       "      <td>We report a case of 38-year-old Greek woman wh...</td>\n",
       "      <td>A 38-year-old Greek woman came to our hospital...</td>\n",
       "      <td>Doctor: Good afternoon, how can I help you tod...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'We report a case of 38-year-old Gree...</td>\n",
       "      <td>167</td>\n",
       "      <td>0.806871</td>\n",
       "      <td>0.918794</td>\n",
       "      <td>0.839161</td>\n",
       "      <td>0.904872</td>\n",
       "      <td>0.904872</td>\n",
       "      <td>0.964090</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.886596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>74937</td>\n",
       "      <td>A 52-year-old right-handed woman with no prior...</td>\n",
       "      <td>A 52-year-old woman with no prior history of n...</td>\n",
       "      <td>Doctor: Good morning, I am Dr. X, what brings ...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 52-year-old right-handed woman wit...</td>\n",
       "      <td>450</td>\n",
       "      <td>0.771265</td>\n",
       "      <td>0.906475</td>\n",
       "      <td>0.853012</td>\n",
       "      <td>0.892086</td>\n",
       "      <td>0.892086</td>\n",
       "      <td>0.959471</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>0.870116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>785</th>\n",
       "      <td>153907</td>\n",
       "      <td>A 78-year-old Caucasian woman with no known pa...</td>\n",
       "      <td>A 78-year-old Caucasian woman with no known pa...</td>\n",
       "      <td>Doctor: Good afternoon, what brings you into t...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 78-year-old Caucasian woman with n...</td>\n",
       "      <td>785</td>\n",
       "      <td>0.758163</td>\n",
       "      <td>0.853403</td>\n",
       "      <td>0.789474</td>\n",
       "      <td>0.842932</td>\n",
       "      <td>0.842932</td>\n",
       "      <td>0.930493</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.829281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>79169</td>\n",
       "      <td>We report the case of a 27-year-old Tunisian m...</td>\n",
       "      <td>A 27-year-old Tunisian man presented with dysp...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'We report the case of a 27-year-old ...</td>\n",
       "      <td>26</td>\n",
       "      <td>0.677541</td>\n",
       "      <td>0.910448</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.870647</td>\n",
       "      <td>0.870647</td>\n",
       "      <td>0.947387</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>0.828577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>150799</td>\n",
       "      <td>A 16-year-old female patient presented with a ...</td>\n",
       "      <td>A 16-year-old female patient presented with a ...</td>\n",
       "      <td>Doctor: Hi, how are you feeling today?\\nPatien...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 16-year-old female patient present...</td>\n",
       "      <td>775</td>\n",
       "      <td>0.716391</td>\n",
       "      <td>0.870536</td>\n",
       "      <td>0.793722</td>\n",
       "      <td>0.834821</td>\n",
       "      <td>0.834821</td>\n",
       "      <td>0.946489</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.819489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>436</th>\n",
       "      <td>191662</td>\n",
       "      <td>A 71-year-old man, known with polycythemia ver...</td>\n",
       "      <td>A 71-year-old man, known with polycythemia ver...</td>\n",
       "      <td>Doctor: Good afternoon, how are you feeling to...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 71-year-old man, known with polycy...</td>\n",
       "      <td>436</td>\n",
       "      <td>0.089849</td>\n",
       "      <td>0.355191</td>\n",
       "      <td>0.131868</td>\n",
       "      <td>0.213115</td>\n",
       "      <td>0.213115</td>\n",
       "      <td>0.812638</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>0.260705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>139346</td>\n",
       "      <td>A 59-year-old Caucasian woman underwent laparo...</td>\n",
       "      <td>A 59-year-old Caucasian woman was admitted to ...</td>\n",
       "      <td>Doctor: Good morning, how are you feeling toda...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 59-year-old Caucasian woman underw...</td>\n",
       "      <td>739</td>\n",
       "      <td>0.099169</td>\n",
       "      <td>0.329193</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.167702</td>\n",
       "      <td>0.167702</td>\n",
       "      <td>0.807469</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.246746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>476</th>\n",
       "      <td>92200</td>\n",
       "      <td>We present here the case of a five year-old bo...</td>\n",
       "      <td>A five-year-old male child was admitted to our...</td>\n",
       "      <td>Doctor: Hello, can you tell me about the patie...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'We present here the case of a five y...</td>\n",
       "      <td>476</td>\n",
       "      <td>0.165677</td>\n",
       "      <td>0.339066</td>\n",
       "      <td>0.172840</td>\n",
       "      <td>0.216216</td>\n",
       "      <td>0.216216</td>\n",
       "      <td>0.810244</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>0.246128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>490</th>\n",
       "      <td>111234</td>\n",
       "      <td>Our patient is an 8-year-old boy, first admitt...</td>\n",
       "      <td>An 8-year-old boy was admitted to our departme...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'Our patient is an 8-year-old boy, fi...</td>\n",
       "      <td>490</td>\n",
       "      <td>0.068527</td>\n",
       "      <td>0.364964</td>\n",
       "      <td>0.102689</td>\n",
       "      <td>0.150852</td>\n",
       "      <td>0.150852</td>\n",
       "      <td>0.815125</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0.214810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>890</th>\n",
       "      <td>85723</td>\n",
       "      <td>A 39-year-old male with skin burns on his righ...</td>\n",
       "      <td>A 39-year-old male was referred to our departm...</td>\n",
       "      <td>Doctor: Hello, I understand you have been refe...</td>\n",
       "      <td>meditron-7b-generator-gold</td>\n",
       "      <td>{'gold': 'A 39-year-old male with skin burns o...</td>\n",
       "      <td>890</td>\n",
       "      <td>0.041262</td>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.087500</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>0.130435</td>\n",
       "      <td>0.775815</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.207940</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "167  170531  We report a case of 38-year-old Greek woman wh...   \n",
       "450   74937  A 52-year-old right-handed woman with no prior...   \n",
       "785  153907  A 78-year-old Caucasian woman with no known pa...   \n",
       "26    79169  We report the case of a 27-year-old Tunisian m...   \n",
       "775  150799  A 16-year-old female patient presented with a ...   \n",
       "..      ...                                                ...   \n",
       "436  191662  A 71-year-old man, known with polycythemia ver...   \n",
       "739  139346  A 59-year-old Caucasian woman underwent laparo...   \n",
       "476   92200  We present here the case of a five year-old bo...   \n",
       "490  111234  Our patient is an 8-year-old boy, first admitt...   \n",
       "890   85723  A 39-year-old male with skin burns on his righ...   \n",
       "\n",
       "                                                  pred  \\\n",
       "167  A 38-year-old Greek woman came to our hospital...   \n",
       "450  A 52-year-old woman with no prior history of n...   \n",
       "785  A 78-year-old Caucasian woman with no known pa...   \n",
       "26   A 27-year-old Tunisian man presented with dysp...   \n",
       "775  A 16-year-old female patient presented with a ...   \n",
       "..                                                 ...   \n",
       "436  A 71-year-old man, known with polycythemia ver...   \n",
       "739  A 59-year-old Caucasian woman was admitted to ...   \n",
       "476  A five-year-old male child was admitted to our...   \n",
       "490  An 8-year-old boy was admitted to our departme...   \n",
       "890  A 39-year-old male was referred to our departm...   \n",
       "\n",
       "                                          conversation  \\\n",
       "167  Doctor: Good afternoon, how can I help you tod...   \n",
       "450  Doctor: Good morning, I am Dr. X, what brings ...   \n",
       "785  Doctor: Good afternoon, what brings you into t...   \n",
       "26   Doctor: Good morning, what brings you here tod...   \n",
       "775  Doctor: Hi, how are you feeling today?\\nPatien...   \n",
       "..                                                 ...   \n",
       "436  Doctor: Good afternoon, how are you feeling to...   \n",
       "739  Doctor: Good morning, how are you feeling toda...   \n",
       "476  Doctor: Hello, can you tell me about the patie...   \n",
       "490  Doctor: Good morning, what brings you here tod...   \n",
       "890  Doctor: Hello, I understand you have been refe...   \n",
       "\n",
       "                     model_name  \\\n",
       "167  meditron-7b-generator-gold   \n",
       "450  meditron-7b-generator-gold   \n",
       "785  meditron-7b-generator-gold   \n",
       "26   meditron-7b-generator-gold   \n",
       "775  meditron-7b-generator-gold   \n",
       "..                          ...   \n",
       "436  meditron-7b-generator-gold   \n",
       "739  meditron-7b-generator-gold   \n",
       "476  meditron-7b-generator-gold   \n",
       "490  meditron-7b-generator-gold   \n",
       "890  meditron-7b-generator-gold   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "167  {'gold': 'We report a case of 38-year-old Gree...   167  0.806871   \n",
       "450  {'gold': 'A 52-year-old right-handed woman wit...   450  0.771265   \n",
       "785  {'gold': 'A 78-year-old Caucasian woman with n...   785  0.758163   \n",
       "26   {'gold': 'We report the case of a 27-year-old ...    26  0.677541   \n",
       "775  {'gold': 'A 16-year-old female patient present...   775  0.716391   \n",
       "..                                                 ...   ...       ...   \n",
       "436  {'gold': 'A 71-year-old man, known with polycy...   436  0.089849   \n",
       "739  {'gold': 'A 59-year-old Caucasian woman underw...   739  0.099169   \n",
       "476  {'gold': 'We present here the case of a five y...   476  0.165677   \n",
       "490  {'gold': 'Our patient is an 8-year-old boy, fi...   490  0.068527   \n",
       "890  {'gold': 'A 39-year-old male with skin burns o...   890  0.041262   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "167  0.918794  0.839161  0.904872   0.904872  0.964090                     9   \n",
       "450  0.906475  0.853012  0.892086   0.892086  0.959471                     9   \n",
       "785  0.853403  0.789474  0.842932   0.842932  0.930493                     9   \n",
       "26   0.910448  0.800000  0.870647   0.870647  0.947387                     9   \n",
       "775  0.870536  0.793722  0.834821   0.834821  0.946489                     8   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "436  0.355191  0.131868  0.213115   0.213115  0.812638                     2   \n",
       "739  0.329193  0.100000  0.167702   0.167702  0.807469                     5   \n",
       "476  0.339066  0.172840  0.216216   0.216216  0.810244                     2   \n",
       "490  0.364964  0.102689  0.150852   0.150852  0.815125                     3   \n",
       "890  0.260870  0.087500  0.130435   0.130435  0.775815                     2   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "167                  9                    9                    10   \n",
       "450                  8                    8                     9   \n",
       "785                  8                    8                    10   \n",
       "26                   6                    6                     9   \n",
       "775                  7                    8                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "436                  7                    7                     8   \n",
       "739                  7                    8                     8   \n",
       "476                  4                    5                     4   \n",
       "490                  6                    6                     4   \n",
       "890                  7                    8                    10   \n",
       "\n",
       "     aggregated_score  \n",
       "167          0.886596  \n",
       "450          0.870116  \n",
       "785          0.829281  \n",
       "26           0.828577  \n",
       "775          0.819489  \n",
       "..                ...  \n",
       "436          0.260705  \n",
       "739          0.246746  \n",
       "476          0.246128  \n",
       "490          0.214810  \n",
       "890          0.207940  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "meditron-13b-generator-gold\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx</th>\n",
       "      <th>gold</th>\n",
       "      <th>pred</th>\n",
       "      <th>conversation</th>\n",
       "      <th>model_name</th>\n",
       "      <th>pairs</th>\n",
       "      <th>idxs</th>\n",
       "      <th>bleu</th>\n",
       "      <th>rouge1</th>\n",
       "      <th>rouge2</th>\n",
       "      <th>rougeL</th>\n",
       "      <th>rougeLsum</th>\n",
       "      <th>bert</th>\n",
       "      <th>gpt_similarity_score</th>\n",
       "      <th>gpt_clarity_score</th>\n",
       "      <th>gpt_coherence_score</th>\n",
       "      <th>gpt_factuality_score</th>\n",
       "      <th>aggregated_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>967</th>\n",
       "      <td>187895</td>\n",
       "      <td>A 32-year old white man presented to our depar...</td>\n",
       "      <td>A 32-year-old white man presented to our depar...</td>\n",
       "      <td>Doctor: Hi, how can I help you today?\\nPatient...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 32-year old white man presented to...</td>\n",
       "      <td>967</td>\n",
       "      <td>0.851272</td>\n",
       "      <td>0.959391</td>\n",
       "      <td>0.913265</td>\n",
       "      <td>0.928934</td>\n",
       "      <td>0.928934</td>\n",
       "      <td>0.981153</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.919680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>83178</td>\n",
       "      <td>A 69-year-old Chinese woman presented with a p...</td>\n",
       "      <td>A 69-year-old Chinese woman presented to the D...</td>\n",
       "      <td>Doctor: Good morning, what brings you here tod...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 69-year-old Chinese woman presente...</td>\n",
       "      <td>537</td>\n",
       "      <td>0.841135</td>\n",
       "      <td>0.938462</td>\n",
       "      <td>0.881443</td>\n",
       "      <td>0.897436</td>\n",
       "      <td>0.897436</td>\n",
       "      <td>0.962045</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.888529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>377</th>\n",
       "      <td>17017</td>\n",
       "      <td>A 40-year-old man presented to the University ...</td>\n",
       "      <td>The patient is a 40-year-old man who presented...</td>\n",
       "      <td>Doctor: Hi, what brings you to the hospital to...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 40-year-old man presented to the U...</td>\n",
       "      <td>377</td>\n",
       "      <td>0.677255</td>\n",
       "      <td>0.874396</td>\n",
       "      <td>0.766990</td>\n",
       "      <td>0.840580</td>\n",
       "      <td>0.840580</td>\n",
       "      <td>0.954826</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>0.824077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>34637</td>\n",
       "      <td>A four-year old Iranian girl was referred to t...</td>\n",
       "      <td>A four-year old girl, the first child of nonco...</td>\n",
       "      <td>Doctor: Hello, how are you today?\\nPatient: I'...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A four-year old Iranian girl was ref...</td>\n",
       "      <td>153</td>\n",
       "      <td>0.602530</td>\n",
       "      <td>0.903073</td>\n",
       "      <td>0.783848</td>\n",
       "      <td>0.874704</td>\n",
       "      <td>0.874704</td>\n",
       "      <td>0.950314</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>0.823903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>197985</td>\n",
       "      <td>A 48-year-old man with a history of alcohol ab...</td>\n",
       "      <td>A 48-year-old man with a history of alcohol ab...</td>\n",
       "      <td>Doctor: Good morning, how can I help you today...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 48-year-old man with a history of ...</td>\n",
       "      <td>999</td>\n",
       "      <td>0.692016</td>\n",
       "      <td>0.853081</td>\n",
       "      <td>0.761905</td>\n",
       "      <td>0.829384</td>\n",
       "      <td>0.829384</td>\n",
       "      <td>0.950866</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.806125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>450</th>\n",
       "      <td>117607</td>\n",
       "      <td>The author reports a case of 30-year-old male ...</td>\n",
       "      <td>A 30-year-old male patient reported to the out...</td>\n",
       "      <td>Doctor: Hello, how are you today?\\nPatient: I'...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'The author reports a case of 30-year...</td>\n",
       "      <td>450</td>\n",
       "      <td>0.073957</td>\n",
       "      <td>0.366366</td>\n",
       "      <td>0.120846</td>\n",
       "      <td>0.198198</td>\n",
       "      <td>0.198198</td>\n",
       "      <td>0.813419</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>0.276974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>44715</td>\n",
       "      <td>Our patient is a 48-year-old female who has a ...</td>\n",
       "      <td>A 48 year old woman with a history of localize...</td>\n",
       "      <td>Doctor: Hello, how are you doing today?\\nPatie...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'Our patient is a 48-year-old female ...</td>\n",
       "      <td>164</td>\n",
       "      <td>0.080597</td>\n",
       "      <td>0.356955</td>\n",
       "      <td>0.137203</td>\n",
       "      <td>0.199475</td>\n",
       "      <td>0.199475</td>\n",
       "      <td>0.812206</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0.273158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>468</th>\n",
       "      <td>129905</td>\n",
       "      <td>A 37-year-old man had a scalp mass on the occi...</td>\n",
       "      <td>A 37-year-old man with neurofibromatosis type ...</td>\n",
       "      <td>Doctor: Hello, what brings you here today?\\nPa...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 37-year-old man had a scalp mass o...</td>\n",
       "      <td>468</td>\n",
       "      <td>0.171211</td>\n",
       "      <td>0.291262</td>\n",
       "      <td>0.214425</td>\n",
       "      <td>0.217476</td>\n",
       "      <td>0.217476</td>\n",
       "      <td>0.766178</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>0.270483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>440</th>\n",
       "      <td>85180</td>\n",
       "      <td>A 54-year-old Caucasian woman with open angle ...</td>\n",
       "      <td>A 54-year-old Caucasian woman was referred to ...</td>\n",
       "      <td>Doctor: Good morning, Mrs. I understand that y...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A 54-year-old Caucasian woman with o...</td>\n",
       "      <td>440</td>\n",
       "      <td>0.044801</td>\n",
       "      <td>0.240000</td>\n",
       "      <td>0.092879</td>\n",
       "      <td>0.147692</td>\n",
       "      <td>0.147692</td>\n",
       "      <td>0.767763</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>0.216086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>203782</td>\n",
       "      <td>A now 76-year-old male underwent buried PD cat...</td>\n",
       "      <td>A now 76-year-old male underwent buried PD cat...</td>\n",
       "      <td>Doctor: Good afternoon, how are you feeling to...</td>\n",
       "      <td>meditron-13b-generator-gold</td>\n",
       "      <td>{'gold': 'A now 76-year-old male underwent bur...</td>\n",
       "      <td>605</td>\n",
       "      <td>0.099083</td>\n",
       "      <td>0.245098</td>\n",
       "      <td>0.113300</td>\n",
       "      <td>0.147059</td>\n",
       "      <td>0.147059</td>\n",
       "      <td>0.779885</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0.197493</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        idx                                               gold  \\\n",
       "967  187895  A 32-year old white man presented to our depar...   \n",
       "537   83178  A 69-year-old Chinese woman presented with a p...   \n",
       "377   17017  A 40-year-old man presented to the University ...   \n",
       "153   34637  A four-year old Iranian girl was referred to t...   \n",
       "999  197985  A 48-year-old man with a history of alcohol ab...   \n",
       "..      ...                                                ...   \n",
       "450  117607  The author reports a case of 30-year-old male ...   \n",
       "164   44715  Our patient is a 48-year-old female who has a ...   \n",
       "468  129905  A 37-year-old man had a scalp mass on the occi...   \n",
       "440   85180  A 54-year-old Caucasian woman with open angle ...   \n",
       "605  203782  A now 76-year-old male underwent buried PD cat...   \n",
       "\n",
       "                                                  pred  \\\n",
       "967  A 32-year-old white man presented to our depar...   \n",
       "537  A 69-year-old Chinese woman presented to the D...   \n",
       "377  The patient is a 40-year-old man who presented...   \n",
       "153  A four-year old girl, the first child of nonco...   \n",
       "999  A 48-year-old man with a history of alcohol ab...   \n",
       "..                                                 ...   \n",
       "450  A 30-year-old male patient reported to the out...   \n",
       "164  A 48 year old woman with a history of localize...   \n",
       "468  A 37-year-old man with neurofibromatosis type ...   \n",
       "440  A 54-year-old Caucasian woman was referred to ...   \n",
       "605  A now 76-year-old male underwent buried PD cat...   \n",
       "\n",
       "                                          conversation  \\\n",
       "967  Doctor: Hi, how can I help you today?\\nPatient...   \n",
       "537  Doctor: Good morning, what brings you here tod...   \n",
       "377  Doctor: Hi, what brings you to the hospital to...   \n",
       "153  Doctor: Hello, how are you today?\\nPatient: I'...   \n",
       "999  Doctor: Good morning, how can I help you today...   \n",
       "..                                                 ...   \n",
       "450  Doctor: Hello, how are you today?\\nPatient: I'...   \n",
       "164  Doctor: Hello, how are you doing today?\\nPatie...   \n",
       "468  Doctor: Hello, what brings you here today?\\nPa...   \n",
       "440  Doctor: Good morning, Mrs. I understand that y...   \n",
       "605  Doctor: Good afternoon, how are you feeling to...   \n",
       "\n",
       "                      model_name  \\\n",
       "967  meditron-13b-generator-gold   \n",
       "537  meditron-13b-generator-gold   \n",
       "377  meditron-13b-generator-gold   \n",
       "153  meditron-13b-generator-gold   \n",
       "999  meditron-13b-generator-gold   \n",
       "..                           ...   \n",
       "450  meditron-13b-generator-gold   \n",
       "164  meditron-13b-generator-gold   \n",
       "468  meditron-13b-generator-gold   \n",
       "440  meditron-13b-generator-gold   \n",
       "605  meditron-13b-generator-gold   \n",
       "\n",
       "                                                 pairs  idxs      bleu  \\\n",
       "967  {'gold': 'A 32-year old white man presented to...   967  0.851272   \n",
       "537  {'gold': 'A 69-year-old Chinese woman presente...   537  0.841135   \n",
       "377  {'gold': 'A 40-year-old man presented to the U...   377  0.677255   \n",
       "153  {'gold': 'A four-year old Iranian girl was ref...   153  0.602530   \n",
       "999  {'gold': 'A 48-year-old man with a history of ...   999  0.692016   \n",
       "..                                                 ...   ...       ...   \n",
       "450  {'gold': 'The author reports a case of 30-year...   450  0.073957   \n",
       "164  {'gold': 'Our patient is a 48-year-old female ...   164  0.080597   \n",
       "468  {'gold': 'A 37-year-old man had a scalp mass o...   468  0.171211   \n",
       "440  {'gold': 'A 54-year-old Caucasian woman with o...   440  0.044801   \n",
       "605  {'gold': 'A now 76-year-old male underwent bur...   605  0.099083   \n",
       "\n",
       "       rouge1    rouge2    rougeL  rougeLsum      bert  gpt_similarity_score  \\\n",
       "967  0.959391  0.913265  0.928934   0.928934  0.981153                     9   \n",
       "537  0.938462  0.881443  0.897436   0.897436  0.962045                     9   \n",
       "377  0.874396  0.766990  0.840580   0.840580  0.954826                     9   \n",
       "153  0.903073  0.783848  0.874704   0.874704  0.950314                     8   \n",
       "999  0.853081  0.761905  0.829384   0.829384  0.950866                    10   \n",
       "..        ...       ...       ...        ...       ...                   ...   \n",
       "450  0.366366  0.120846  0.198198   0.198198  0.813419                     7   \n",
       "164  0.356955  0.137203  0.199475   0.199475  0.812206                     6   \n",
       "468  0.291262  0.214425  0.217476   0.217476  0.766178                     2   \n",
       "440  0.240000  0.092879  0.147692   0.147692  0.767763                     1   \n",
       "605  0.245098  0.113300  0.147059   0.147059  0.779885                     3   \n",
       "\n",
       "     gpt_clarity_score  gpt_coherence_score  gpt_factuality_score  \\\n",
       "967                  9                    8                    10   \n",
       "537                  7                    8                    10   \n",
       "377                  9                    8                    10   \n",
       "153                  8                    8                     7   \n",
       "999                  6                    7                    10   \n",
       "..                 ...                  ...                   ...   \n",
       "450                  7                    7                    10   \n",
       "164                  7                    6                    10   \n",
       "468                  6                    7                     7   \n",
       "440                  8                    9                    10   \n",
       "605                  7                    7                     1   \n",
       "\n",
       "     aggregated_score  \n",
       "967          0.919680  \n",
       "537          0.888529  \n",
       "377          0.824077  \n",
       "153          0.823903  \n",
       "999          0.806125  \n",
       "..                ...  \n",
       "450          0.276974  \n",
       "164          0.273158  \n",
       "468          0.270483  \n",
       "440          0.216086  \n",
       "605          0.197493  \n",
       "\n",
       "[1000 rows x 18 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "note_eval_res = {}\n",
    "evaluation_path = 'evaluation'\n",
    "for model in NOTE_MODELS:\n",
    "    note_eval_res[model] = load_file(f\"{evaluation_path}/{model}_eval_res/all_scores.jsonl\")\n",
    "    print(f\"\\n\\n{model}\\n\")\n",
    "    display(note_eval_res[model].sort_values(by='aggregated_score', ascending=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Elo ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation directory exists, loading...\n",
      "Progress: {'create eval directory': 'done', 'pairs_idx': 'done', 'scores': 'done', 'compute elos': 'tbd', 'gpt_rank': 'done'}\n",
      "GPT-4 ranking...\n",
      "GPT-4 ranks already computed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Computing ELO rankings: 100%|██████████| 3990/3990 [00:04<00:00, 934.02it/s] \n"
     ]
    }
   ],
   "source": [
    "elo_rankings = elo_ranking(path='evaluation/elo_inputs.jsonl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>elo_history</th>\n",
       "      <th>score_histories</th>\n",
       "      <th>final_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>meditron-7b-direct-trunc</td>\n",
       "      <td>[(None, 1000), (5358, 999.5), (6023, 1000.0021...</td>\n",
       "      <td>[1000, 999.5, 1000.0021565891029, 999.50286786...</td>\n",
       "      <td>1037.372550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>meditron-13b-generator</td>\n",
       "      <td>[(None, 1000), (151012, 1000.5), (87643, 1001....</td>\n",
       "      <td>[1000, 1000.5, 1001.0007082097612, 1001.496398...</td>\n",
       "      <td>1031.225550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>meditron-13b-direct-trunc</td>\n",
       "      <td>[(None, 1000), (151012, 999.5), (131638, 998.9...</td>\n",
       "      <td>[1000, 999.5, 998.9992825136991, 999.501429839...</td>\n",
       "      <td>1030.897057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>meditron-7b-generator</td>\n",
       "      <td>[(None, 1000), (72604, 999.5), (54292, 999.001...</td>\n",
       "      <td>[1000, 999.5, 999.0014391117091, 999.502156598...</td>\n",
       "      <td>1026.303979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>llama-2-7b-direct</td>\n",
       "      <td>[(None, 1000), (5358, 1000.5), (54292, 1000.99...</td>\n",
       "      <td>[1000, 1000.5, 1000.9985608882909, 1000.496404...</td>\n",
       "      <td>992.804541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gpt3-direct</td>\n",
       "      <td>[(None, 1000), (150522, 1000.5), (138662, 1001...</td>\n",
       "      <td>[1000, 1000.5, 1001.0, 1001.4978433975539, 100...</td>\n",
       "      <td>987.553568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mistral-7b-direct</td>\n",
       "      <td>[(None, 1000), (72604, 1000.5), (138662, 1000....</td>\n",
       "      <td>[1000, 1000.5, 1000.0, 1000.4985650139779, 999...</td>\n",
       "      <td>978.525903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>llama-2-13b-direct</td>\n",
       "      <td>[(None, 1000), (150522, 999.5), (21023, 1000.0...</td>\n",
       "      <td>[1000, 999.5, 1000.0014370466154, 1000.5035853...</td>\n",
       "      <td>915.316853</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       model  \\\n",
       "1   meditron-7b-direct-trunc   \n",
       "3     meditron-13b-generator   \n",
       "2  meditron-13b-direct-trunc   \n",
       "6      meditron-7b-generator   \n",
       "4          llama-2-7b-direct   \n",
       "7                gpt3-direct   \n",
       "5          mistral-7b-direct   \n",
       "0         llama-2-13b-direct   \n",
       "\n",
       "                                         elo_history  \\\n",
       "1  [(None, 1000), (5358, 999.5), (6023, 1000.0021...   \n",
       "3  [(None, 1000), (151012, 1000.5), (87643, 1001....   \n",
       "2  [(None, 1000), (151012, 999.5), (131638, 998.9...   \n",
       "6  [(None, 1000), (72604, 999.5), (54292, 999.001...   \n",
       "4  [(None, 1000), (5358, 1000.5), (54292, 1000.99...   \n",
       "7  [(None, 1000), (150522, 1000.5), (138662, 1001...   \n",
       "5  [(None, 1000), (72604, 1000.5), (138662, 1000....   \n",
       "0  [(None, 1000), (150522, 999.5), (21023, 1000.0...   \n",
       "\n",
       "                                     score_histories  final_score  \n",
       "1  [1000, 999.5, 1000.0021565891029, 999.50286786...  1037.372550  \n",
       "3  [1000, 1000.5, 1001.0007082097612, 1001.496398...  1031.225550  \n",
       "2  [1000, 999.5, 998.9992825136991, 999.501429839...  1030.897057  \n",
       "6  [1000, 999.5, 999.0014391117091, 999.502156598...  1026.303979  \n",
       "4  [1000, 1000.5, 1000.9985608882909, 1000.496404...   992.804541  \n",
       "7  [1000, 1000.5, 1001.0, 1001.4978433975539, 100...   987.553568  \n",
       "5  [1000, 1000.5, 1000.0, 1000.4985650139779, 999...   978.525903  \n",
       "0  [1000, 999.5, 1000.0014370466154, 1000.5035853...   915.316853  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(elo_rankings.sort_values(by='final_score', ascending=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHHCAYAAABeLEexAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOydd3gUVduH79maTe+90nvvvfeqqBQFbKDoa9dXFBWxg6+fXbEhAgpIBwHpvUhLQidAgPTek822+f6YZJPNbkJAmjL3deVi55wzZ84sye5vnvMUQRRFERkZGRkZGRmZuxjF7V6AjIyMjIyMjMztRhZEMjIyMjIyMnc9siCSkZGRkZGRueuRBZGMjIyMjIzMXY8siGRkZGRkZGTuemRBJCMjIyMjI3PXIwsiGRkZGRkZmbseWRDJyMjIyMjI3PXIgkhGRkZGRkbmrkcWRDIyMgAIgsDMmTNv9zLuKC5duoQgCHz88cdXHTtz5kwEQbgFq6o9kydPJjIy8nYvQ0bmH4EsiGRk/sX8/PPPCIJQ7c+BAwdu9xLvapKTk5k5cybR0dG3eykyMnc9qtu9ABkZmZvPrFmziIqKsmuvV6/ebVjNv5MZM2bw6quvXtM5ycnJvP3220RGRtKqVasbvqbvv/8ei8Vyw+eVkfk3IgsiGZm7gMGDB9OuXbvbvYxbislkwmKxoNFobsn1VCoVKtWd8ZFaVFSEi4sLarX6di9FRuYfg7xlJiMjUy3Hjh1j8ODBuLu74+rqSt++fWu9zbZ48WLatm2Lm5sb7u7uNG/enM8++8xmTG5uLs8//zyRkZFotVpCQ0OZOHEimZmZ1jHp6ek8+uijBAQE4OTkRMuWLZk/f77NPJV9fT799FPq1q2LVqvl1KlTAJw5c4YxY8bg7e2Nk5MT7dq1Y82aNdf0Xnz33XfWedu3b8+hQ4ds+h35EG3evJlu3brh6emJq6srDRs25LXXXgNgx44dtG/fHoCHH37Yuo35888/W8///fffadu2LTqdDl9fXx588EGSkpJsrjF58mRcXV25cOECQ4YMwc3NjQkTJlj7qvoQWSwWPv30U5o2bYqTkxMBAQFMnTqVnJwcm3GHDx9m4MCB+Pr6otPpiIqK4pFHHrmm90xG5p/EnfE4IyMjc1PJy8uzERkgOVH7+PhUe87Jkyfp3r077u7uvPLKK6jVaubOnUuvXr3YuXMnHTt2rPbczZs3M27cOPr27ctHH30EwOnTp9m7dy/PPvssAIWFhXTv3p3Tp0/zyCOP0KZNGzIzM1mzZg2JiYn4+vpSUlJCr169OH/+PE8//TRRUVH8/vvvTJ48mdzcXOtc5cybNw+9Xs+UKVPQarV4e3tz8uRJunbtSkhICK+++iouLi4sXbqUUaNGsXz5ckaPHn3V9+/XX3+loKCAqVOnIggCs2fP5p577uHixYvVWmFOnjzJsGHDaNGiBbNmzUKr1XL+/Hn27t0LQOPGjZk1axZvvvkmU6ZMoXv37gB06dIFkPy/Hn74Ydq3b88HH3xAWloan332GXv37uXYsWN4enpar2UymRg4cCDdunXj448/xtnZudp7mTp1qnXuZ555hvj4eL788kuOHTvG3r17UavVpKenM2DAAPz8/Hj11Vfx9PTk0qVLrFix4qrvlYzMPxZRRkbmX8u8efNEwOGPVqu1GQuIb731lvV41KhRokajES9cuGBtS05OFt3c3MQePXrUeN1nn31WdHd3F00mU7Vj3nzzTREQV6xYYddnsVhEURTFTz/9VATEhQsXWvsMBoPYuXNn0dXVVczPzxdFURTj4+NFQHR3dxfT09Nt5urbt6/YvHlzUa/X28zfpUsXsX79+jXeR/m8Pj4+YnZ2trV99erVIiCuXbvW2vbWW2+JlT9S/+///k8ExIyMjGrnP3TokAiI8+bNs2k3GAyiv7+/2KxZM7GkpMTavm7dOhEQ33zzTWvbpEmTREB89dVX7eafNGmSGBERYT3evXu3CIiLFi2yGbdx40ab9pUrV4qAeOjQoWrXLiPzb0PeMpORuQv46quv2Lx5s83Phg0bqh1vNpvZtGkTo0aNok6dOtb2oKAgxo8fz549e8jPz6/2fE9PT4qKiti8eXO1Y5YvX07Lli0dWmjKt57Wr19PYGAg48aNs/ap1WqeeeYZCgsL2blzp8159957L35+ftbj7Oxstm3bxv33309BQQGZmZlkZmaSlZXFwIEDiYuLs9uCcsQDDzyAl5eX9bjcmnPx4sVqzym34KxevfqaHZsPHz5Meno606ZNw8nJydo+dOhQGjVqxB9//GF3zpNPPnnVeX///Xc8PDzo37+/9b3IzMykbdu2uLq6sn37dpu1r1u3DqPReE1rl5H5pyILIhmZu4AOHTrQr18/m5/evXtXOz4jI4Pi4mIaNmxo19e4cWMsFgsJCQnVnj9t2jQaNGjA4MGDCQ0N5ZFHHmHjxo02Yy5cuECzZs1qXPfly5epX78+CoXtR1Xjxo2t/ZWpGkl3/vx5RFHkjTfewM/Pz+bnrbfeAiQfpasRHh5uc1wujqr63VTmgQceoGvXrjz22GMEBAQwduxYli5dWitxVH5fjt7/Ro0a2d23SqUiNDT0qvPGxcWRl5eHv7+/3ftRWFhofS969uzJvffey9tvv42vry8jR45k3rx5lJaWXvUaMjL/VGQfIhkZmRuOv78/0dHR/Pnnn2zYsIENGzYwb948Jk6caOcQfSPR6XQ2x+Xi46WXXmLgwIEOz6lN6gGlUumwXRTFGteya9cutm/fzh9//MHGjRtZsmQJffr0YdOmTdXOeT1otVo70egIi8WCv78/ixYtcthfbl0TBIFly5Zx4MAB1q5dy59//skjjzzC//73Pw4cOICrq+sNW7uMzJ2CLIhkZGTs8PPzw9nZmbNnz9r1nTlzBoVCQVhYWI1zaDQahg8fzvDhw7FYLEybNo25c+fyxhtvUK9ePerWrcuJEydqnCMiIoLY2FgsFovNF/6ZM2es/TVRvt2nVqvp169fjWNvBgqFgr59+9K3b18++eQT3n//fV5//XW2b99Ov379qs1sXX5fZ8+epU+fPjZ9Z8+evep9V0fdunXZsmULXbt2tROPjujUqROdOnXivffe49dff2XChAksXryYxx577LquLyNzJyNvmcnIyNihVCoZMGAAq1ev5tKlS9b2tLQ0fv31V7p164a7u3u152dlZdkcKxQKWrRoAWDddrn33nuJiYlh5cqVdueXW16GDBlCamoqS5YssfaZTCa++OILXF1d6dmzZ4334e/vT69evZg7dy4pKSl2/RkZGTWe/3fIzs62aytPvlj+Hri4uABS+oHKtGvXDn9/f7799lubbaoNGzZw+vRphg4del1ruv/++zGbzbzzzjt2fSaTybqOnJwcO+tX1bXLyPzbkC1EMjJ3ARs2bLBaVSrTpUsXG6fpyrz77rvWPDrTpk1DpVIxd+5cSktLmT17do3Xe+yxx8jOzqZPnz6EhoZy+fJlvvjiC1q1amX1/3n55ZdZtmwZ9913H4888ght27YlOzubNWvW8O2339KyZUumTJnC3LlzmTx5MkeOHCEyMpJly5axd+9ePv30U9zc3K5671999RXdunWjefPmPP7449SpU4e0tDT2799PYmIiMTExtXgHr51Zs2axa9cuhg4dSkREBOnp6Xz99deEhobSrVs3QLLYeHp68u233+Lm5oaLiwsdO3YkKiqKjz76iIcffpiePXsybtw4a9h9ZGQkzz///HWtqWfPnkydOpUPPviA6OhoBgwYgFqtJi4ujt9//53PPvuMMWPGMH/+fL7++mtGjx5N3bp1KSgo4Pvvv8fd3Z0hQ4bcyLdJRubO4fYGucnIyNxMagq7p0q4N1XC7kVRFI8ePSoOHDhQdHV1FZ2dncXevXuL+/btu+p1ly1bJg4YMED09/cXNRqNGB4eLk6dOlVMSUmxGZeVlSU+/fTTYkhIiKjRaMTQ0FBx0qRJYmZmpnVMWlqa+PDDD4u+vr6iRqMRmzdvbhemXh4eP2fOHIfruXDhgjhx4kQxMDBQVKvVYkhIiDhs2DBx2bJlNd5HTfNWfb+qht1v3bpVHDlypBgcHCxqNBoxODhYHDdunHju3DmbeVavXi02adJEVKlUdv8nS5YsEVu3bi1qtVrR29tbnDBhgpiYmGhz/qRJk0QXFxeH668adl/Od999J7Zt21bU6XSim5ub2Lx5c/GVV14Rk5OTRVGU/t/HjRsnhoeHi1qtVvT39xeHDRsmHj58uMb3S0bmn4wgijV4BcrIyMjIyMjI3AXIPkQyMjIyMjIydz2yIJKRkZGRkZG565EFkYyMjIyMjMxdjyyIZGRkZGRkZO56ZEEkIyMjIyMjc9cjCyIZGRkZGRmZux45MWMtsFgsJCcn4+bmVm2qfRkZGRkZGZk7C1EUKSgoIDg4+Kr1/mRBVAuSk5OvWrdJRkZGRkZG5s4kISGB0NDQGsfIgqgWlJcHSEhIqLF+k4yMjIyMjMydQ35+PmFhYbUq8yMLolpQvk3m7u4uCyIZGRkZGZl/GLVxd5GdqmVkZGRkZGTuemRBJCMjIyMjI3PXIwsiGRkZGRkZmbseWRDJyMjIyMjI3PXIgkhGRkZGRkbmrkcWRDIyMjIyMjJ3PbIgkpGRkZGRkbnrkQWRjIyMjIyMzF3PbRVEu3btYvjw4QQHByMIAqtWrbLpF0WRN998k6CgIHQ6Hf369SMuLs7hXKWlpbRq1QpBEIiOjrbpi42NpXv37jg5OREWFsbs2bNv0h3JyMjIyMjI/BO5rYKoqKiIli1b8tVXXznsnz17Np9//jnffvstBw8exMXFhYEDB6LX6+3GvvLKKwQHB9u15+fnM2DAACIiIjhy5Ahz5sxh5syZfPfddzf8fmRkZGRkZGT+mdzW0h2DBw9m8ODBDvtEUeTTTz9lxowZjBw5EoBffvmFgIAAVq1axdixY61jN2zYwKZNm1i+fDkbNmywmWfRokUYDAZ++uknNBoNTZs2JTo6mk8++YQpU6bcvJuTkZGRkZGR+cdwx/oQxcfHk5qaSr9+/axtHh4edOzYkf3791vb0tLSePzxx1mwYAHOzs528+zfv58ePXqg0WisbQMHDuTs2bPk5OQ4vHZpaSn5+fk2PzIyMjIyMjL/Xu5YQZSamgpAQECATXtAQIC1TxRFJk+ezBNPPEG7du2qncfRHJWvUZUPPvgADw8P609YWNjfuhcZGRkZGZlaYzaBxXy7V3HXcccKotrwxRdfUFBQwPTp02/ovNOnTycvL8/6k5CQcEPnl5GRkbkT2ZW4i4WnFiKK4u1eyt2LsQS+aAOzvGHHh7d7NXcVd6wgCgwMBKQtscqkpaVZ+7Zt28b+/fvRarWoVCrq1asHQLt27Zg0aZJ1HkdzVL5GVbRaLe7u7jY/MjIyMv9m8krzeGrrU3x06CP+Sv3rdi/n7uXcn5B7WXq94wMozr6967mLuGMFUVRUFIGBgWzdutXalp+fz8GDB+ncuTMAn3/+OTExMURHRxMdHc369esBWLJkCe+99x4AnTt3ZteuXRiNRus8mzdvpmHDhnh5ed3CO5KRkZG5c1l9frX19a7EXbdxJXc5J5bZHs+Ogux46fXu/8GnLSDrwq1f113AbRVEhYWFVjEDkiN1dHQ0V65cQRAEnnvuOd59913WrFnD8ePHmThxIsHBwYwaNQqA8PBwmjVrZv1p0KABAHXr1iU0NBSA8ePHo9FoePTRRzl58iRLlizhs88+44UXXrgdtywjIyNzR7Hs3DL+u+u/zDk8x9omC6LbRFGWZCGqyoFvQJ8PW2dJ1qMv20H8bjAZYOs7cHbjrV/rv5DbGnZ/+PBhevfubT0uFymTJk3i559/5pVXXqGoqIgpU6aQm5tLt27d2LhxI05OTrW+hoeHB5s2beKpp56ibdu2+Pr68uabb8oh9zIyMnc9F3Iv8Pb+t+3aL+VfYtqWaQyOGszwusNvw8pqZsaeGWTqM/m89+dolJpqx2UUZ7D6wmrGNhyLq8YVgFx9LkqFEjeNm8NzLuVdIqEggaa+TfF28r4p66+W6EVgNkBQKxj1DXwj7YYQv8vWciRaYP4w23PfyALlbf1Krz2mUlBqQBBu90psEETZe+6q5Ofn4+HhQV5enuxPJCMj86/hi2Nf8F1sRZLaZj7NcNW4ciDlgLXtyINHahQdtwKj2cjR9KPU9azLhD8mkFyUDMCXfb6kZ1jPas8btWoUF/Kk7SU/nR//af0fPj36Ka5qV1aPWo1KYSsgSkwldFjUwXp8y+7dbITt78OeT6Tj4Z9B28mQfhq+7lS7OR7eAOGdpei0O1UYmY2SmNs5G06tgl7TofmYm3rJa/n+vmN9iGRkZGRkbh6iKLL+4nqbtre6vEWP0B42bb2W9KLYWHwrl2bH58c+57FNj9F7aW+rGAJ4etvTdPq1E7+c/MVmfLGxmN2Ju61iCCCjJIM3971Jtj6bKwVXOJJ2xO46OxJ22BwfTDl4Q++jWo78XCGGVDpoViYS/BuDi7/t2E5POZ7j5Eppa+0dX7iw3b7fYpbC+W8XogjzBsNnLeHwT5B1/vatpRpkQSQjIyNzFxKbGUtiYSI6lY6D4w9yfNJxGnk3oldYL5txBcYCfjzx420LxTeajfx88udq+4uMRcw5PId9SfsASMhPYOKGiUzbOq3GeTfEb8AiWqzHoijaXWdbwrbrXvc1Eb2o4nVkV9C6Vhw3u7fidWh7GPQ+vJ4G0w5Al/9A8/ulvr++gz+nAyIsHg+GIqldFCFmCXzaHD6uB/kVgvKWknhI+ilIgZJsSfg1GHR71lINsiCSkZGRuUsoMZUwYf0Ent76NA+ufxCAPuF9cFZXZPkPcwtjbv+5jG1YUR7pu9jv+OXUL3bz3Qp2J+22a5vcdLJd29QtU1l9fjVDVg7hbM7ZaueLcI8AYHncct7Y+4a1/VDqIU5lnUKn0jGryywAtl/Zzu7E3eToHVc1uC5KC+HMHxWJF5OPST8AGlcYXKX4eN83oP3j0GQU3L9AalM7SdajAe/C6LmgrbIVZCyGpROl6LSPImHlFMhPgpIciF1y4+7lWjj+u+1xo6G2wu8O4A7daJSRkZGRudFsvryZ2IxYm7ahUUPtxnUJ7kLnIMmhd/HZxQB8fPhjJjWddPMXWYW1F9YCMLbhWHx1vrTwa0Hn4M4UGApYHrfcZuyMvTNsjrVKLT8O/JG1F9bybJtncVG7YLQYabdQqmyw5sIamvg0YXDUYNZdXAfA0DpDGVZnGLMPzSZLn8W0rdPoFdqLL/p+AUB0ejTPbHuGKS2m8GCTB6/tZjLj4Lex0nZR56clUXPkZ6mv2RgY86P9ORoXGPpx9XMqFDDyK1j6kG37+S3wQz/Q59q2x22Gbs9f27r/DqIoWcD+qlJQvcUDt24NtUR2qq4FslO1jIzMv4Epm6awP6WiFmR9r/osH74coZpoH1EUGbFqBJfyLwGw4Z4NhLqF3oqlAlKyyF5Le2GymFgxYgX1vepb+woNhRxMOUiXkC42jtDltPJrxayus4jyiLLr+/CvD1l0epFNm0qhwmQx8dPAn2gf2J4Xd7zIpsubrP37xu3DTeNGj8U9yCmVLEaxE2Orfe8AMBTDwnvAKxKGfgLvBzkcZgQ+7/4wTesMYlDUdW4jZV2AtJMQvxMO/VD9OEEBL50HFx/pOPEwlORCVHdQaa/v2jVxdiP8ViZ+tO7gUxcUKskJXKm+8derguxULSMjIyMDgNliJq80j9SiVJvoMYDv+n9X4xe6IAgsH7GcZj7NABi8YjCv73kd8y2qs7UxfiMmi4nG3o1txBCAq8aVvhF90al0fNff1vqwaMgiFgxZ4FAMAbzS/hUGRw22aTNZTPjr/Gnj3waAvuF9bfrXXlhLu4XtrGIIYPah2exOtN/Ss3J2PVzZDzG/wV9zqx22ycWZnxO38ta+tyg1l1Y/X0341IUmI2DwHOj+km1fnxkwMw8CmktRXnF/SnmNlj8GP/SFRffC/xqBxeJ47uulOBv2f1lxHNYRpuyAx7bcEjF0rciCSEZGRuZfytnsszy26TG6Le5G/2X9ERGp71Wfln4tebjZw/jqfK86h0apoX9kf+vxmgtrWHNhzc1cdsW1LkrXuVoupM7BnZnaYioAjzZ7lBZ+LWocrxAU/Lf9f+3yDA2tOxSlQgnAoKhBvNTuJatA+uCvD+zEysLTC5m2dRr/d+T/SC9OJ6kwyfZCJypt6W2ZabeOQ05a1ro6sy6qLQDFpmIOJB+wG3dNKBS221EDP4AeL0uvGw2R/j33J2x929avpyQbEm5AVN3RBdK9ZsbBZ63gUiXBOOCdvz//TUTeMqsF8paZjIzMPwmjxchPx3/iy+gv7fre7vI299S/55rmu5J/haErK3yNBkQM4P3u7xObEUu7gHY1bxs5QBRFNsRvoJlvM9QKNQEuASgE2+fzi3kXGblqJEpByZb7tlxVvJXnKmoX0M4qamrDrsRdPLVVCmVfNXIVdT3r2vQfSz/GxA0TbdoCnANIK7atkVm+5VbPsx4rRqxA0OfCnPpgMdqMM03dzVen5nEi5RAHzHl26xlVbxTvdL0BwmH+CMg8B0/sAZey9+7KQfhpADh5AiLoq1y/0zQY9EGlxZZCbgIc+h7aTIKAJjVeUry4E+GXEY4738q9LYkY5S0zGRkZmbuYL4594VAMAfSP6O+wvSbC3cNZNGSRdets65WtvLzzZR758xGWxS2zG//j8R/5z7b/VJu/aM2FNfx3938ZunIoA5YPoOUvLdmTtMdmzKJTko9P99DutbJkqZVqOgZ1vCYxBNA5qDO9QnsxtuFYOzEE0MK3hU1W6ydbPsn3A763G2eySDl+zuee53T2aTi1RhJDrhVFxI+HNKP1pgn8kLjFTgy5a6Qv6+0J261z/S0eWgXPnagQQwAhbSU/Hn1uhRiq1w/uKbuf02slJ2iLWSoT8q4/5i/b8ufx+eQturfqFQCIz4tnZdxKNl3aRIvdT/NkgB/GqoOGfnLHZaV2hBxlJiMjI/MPYV/SPsLdw2t0bBZFkXkn5jnsG9twbLUlK65GC78W/DLkF7r82gW9Wc/2BCn53+y/ZnNfg/sAqSzG/evuJ6UoBYA/4v+w9lVm8ZnFdm1PbnmSP0b/Qbh7OKXmUjbEbwDgwcbXGMl1jaiVamsEmSOUCiX7xu1jT9IeXNWutPJvBcDrHV9HQOBS/iUWnl5oc87OhB00OTCfbc46xOZD6Bu9EkpyeM7ZAtVonWmtpjE3Zi45pTkcSTtCx6COf+/GFArsbB5KFUR2h7N/SMetJsCoryXnb6UG8hKkCLi0k2w68jU/BgfgZbaw11lHv6Ji/s+ol0L+kX7Pfjj+A58f+9zmEnucdcz3cOexvHyp4eGNENH5793LLUIWRDIyMjL/AA6kHGDqlqnU96rPihErqh0XnRFtfa1Valk7ai1BrkFklWThofX4W2so3966nH/Z2mawGMjV57I7aTev7XnNZvys/bPwdvK2OiiLosjhtMOczDrpcP7ybblxjcZRYCwgwDmA9oHt/9aabxTdQrrZHI9tJOVpupB7gdUXVqNT6Wjq05TtCdvZfnEDdYou81KAH6RuYfmY75gT/TnphZcdTY2r2pUhUUM4l3OOFXEr2HJ5y98XRNVRt7dVEKU2GsTHOyU/qbHhnTiacpDIX+/FJ/syL0aF25y2xcWZsWvuoXFwR7oFd8Nd624nhsr5zNuTz7w9eRc/Rv5DxBDIPkS1QvYhkpGRud1M3z3dmivng+4fEOoaarVWVGbmvpksj1tO77DezOwy84YXKN2XvI+pm6fatL3b9V27HECV6RHag1NZpxgYOdAu3D3SPdIa1l+Vh5s+zAvtXvjba77ZWEQLnNtETkkGvWPmIFLz1+o3/b6hU1AnDGYDuaW5mEUzYW5hVn8mf50/m+/bbOdXdUMoSIUvO4BXOBPCwonNPG7T7WcyoQRSVTXbS+5rcB+/n/u9xjEAxx46hkqhwmIwIOr1KG/xd+i1fH/LgqgWyIJIRkbmdlJsLKbX0l6UmEps2jeP2UygS4WPSl5pHv2X9afEVGLNp3Oz1pNalMr6+PXMjZ1LlEcU8Xnx1v4X2r7AsnPLuFJwpdo53uj0Bt5O3rQNaMvRtKM8t+M5uzG/D/+dRt6NbsYt3FgK0+GTxmAx8WCrPsTkVV+na0bHGTzQyHFSQoPZQI8lPSgyFrFg8AKOZx6nuW9zh8L3ehBFkY8Pf8y+xN10D+7CvDOLrn5SNagEFSbRdv9vVsQI5uXEEl9F4L6tHUOLn/ZhvHIF9xHD8X38cbT1bdMo3Cxkp2oZGRmZfziiKFrrh+1M3GknhqDCAVdv0gPw56U/KTGV0MCrAe0C2t20tTmrnanjWYc+4X0ArGJIq9Tyy+BfeLjZw/xxzx/Vnu+h9WB43eH0i+iHl5MXfSP6cnzScZYNt3XQbujV8Kbdww3DUAR7P4MyR+gO6ZeqHdoxsGONKQQ0So21uO5/tv2H2Ydm89CGh2xqrv0dojOi+eXUL5zPj6+VGJo/aD7b7tvGwdEb6Ftk6yBvEk14mc0cjb/C52kZPNn0EUb0mMU3/b9lXKNx1nENE0Qaz1yM8YokjvPXrOXi8BGY8/NvyD3dSGRBJCMjI3MHMufwHDr/1pmLeRf546IkLu6tbxvps/bCWp7d/iy9lvYiqTDJOm5E3RHXHAp/PTT2bmxjofq458e09m9tPf6g+wd253QN7sraUWvRqXR2fQ29G/JW57cAyYJ0K+7hb7PkIZvkg/elXSbYaOJBrxYcfvAwG+/dSJfgLkzvMJ0fBv5gUzfOEf3C+wGQW5prbbt3zb08v/15u+gzs8VsFc0Xcy9iNNvFd1k5lHrILn0ASP5aCwYvwEvrZdP+cruXaRPQBj9nP5zdQ3l/zDq6FpcQbqy4Rv+iYtRAb/f6TGv3PEqFkhDXEF7r+BrLRywnOEvk8T8dJ/E816Ej6f/7BNF0AyLqbhDyllktkLfMZGRkbiV5pXn0WdoHg8XAmAZjWBW3CpNoYvXI1aw8L4U4JxfZVi2v51mP87nnERDYNGaTjVC5mbx/8H1+O/MbnlpPtt2/DbXCNgNxsbGYhIIEXtr5ElcKrrB02FIaeldv+RFFkWx9Nj46n5u99L9P2kn4pot9u1ILL54B52v33yo2FtPxV8cO1T8O+JEOQVKZkmx9NmPXjSXMLYxR9Ubx2p7XeKjJQ7zS/hVEUWRHwg5a+LXAR+eD3qSn/SL77VOVQsXGezYS4BIASMV/fzj+A4MjB1PPq579AlY+QXHsYnqEh1CqUPBTShrt9aXw5H6bHEWiKJK3fDkpM96wOT0mUqDlJXvJEfj221hKivG8916UbtcXBVkdsg/RDUYWRDL/dC7mXiSlKIWuIV1v91JkkMLTEwoSaO7X3GH/7+d+Z9Z+qeK6gICISCPvRvw+vMKJtfl8x+d2DOzIDwNrqGV1gzmfc54ntjzBpKaTeKjJQ9WOSy1KJa80r0Yx9I/jjxcr6obVHwBFmZB8VMoUfc93NZ9bA5M2TOJo+lGHffMGzqNdYDu+OPYF38XaX2PH/TtYH7+e2Ydm0yO0B1/1/Yp1F9cxffd065joh6KvOV8TAElH4Yd+bNdpSFYpGevZAUObt3Bq2dLGmpezeAmpM2fanHr/q0oQBOonibz3SzWlXwSBBof+Qunqeu1rqwbZh0hGRsaKyWLinjX38MSWJziecfzqJ8jcdF7c+SLj14/nr5S/HPaXV3gHrBFLVavSz+kxx+G5I+uNvEGrrB31vOqx5b4tNYohgECXwH+PGMq+CPG74FiZH07vGVJywz4zIKoH9Pzv35r+3W7v8kizR1h/z3q7vv878n+Iosifl/50eG6vpb2YfWg2IGXh3pGwg4/++sja/3rH169PDAGEtIFnjtL72fNMeCGJtLjGXBo7jjONm2DKyrIOy5xrW7fN983Xean9y9xT/x7iQgR+7luN9FAobqgYulZkQSQj8y9nX/I+zKL0RDZ+/XiWnFmC2WJmX9I+2i5oy/Jzy23GW0QLl/MvIxuPbw6JBYn8lSoJoaoJ/QAS8hM4ln4MhaCwFicVEOyqoA+KGsT+cftZOmwpHQOlLRZ3jTv9Ivrd5Du4i7GYYecc+Lw1zB8OphLwbwo9XgKdJ9TrC5PWSoVW/wZhbmE83/Z5wtzCmN5hOkOihlj7LhdcZtaBWTa5oGriP9v+Y/VH2jxmM/dHjKRw1y5E8/UV6DXhgShoMBcWkrdypbU96zvJWmW4cgVTipSYU+XvT511a/Eb/yCTmk7iyZZPArC+g4L7p6sQ/lyIOjjYOkfw7I+4nciJGWVk/uGczDxJSlGK9Ytwf/J+VsStYHDUYDoFdWLdhXU24989+C7FpmL2JO3BYDHw25nfuLdBhbPuV9FfWU3x3w/4nk5BnW7dzfzL0Zv0/HLqF+vx8czjWEQLCkHByriVLDm7xOps3CmoE52COvHJkU9oH9jeoU+Qq8aVxj6N+WHgD9YyGY6clWX+JtkXpUgyrTvsq5KMsPWEm1qWYnzj8YxvPJ63Or9F18VdySvNY9k5KRqvf0R/fJx8+OPiH7zT9R2HqQvK8XHyIdAlkOQZM8hbtpyA6a/iPWnSNa1Ff/Ys8SNHOewrPiJt8eWtWg2AS7duhP9gW+Ik0CWQ2T1m88quVxgcOZhGEW1h21ZEkwljSgqasLBrWs+NRvYhqgWyD5HMnYjJYuKr6K/44bjkw7BwyEIaezem7cK21zxXeT4bo9lIm4VtbPoWDF5ww/Kg3M3oTXqGrxpOalGqTbubxo2N926k62+2/l0fdP+AARED+PX0r/SL6FdjuQ6Zm8znbSD7gn27Rxg8Ey2VxLgFPLnlSZuab7N7zGZw1GBEUUQQBDbEb+CVXa8A8GnvT3lu+3PWsT8M+IH2Pq0526Klta3xmdN21yg5fhxjcgqu3buhP32aov0H8J06BUGtJunFl8j/wzadgsLZGUuxJMZ9pkyxWoqC58zBY/iwG3bv18u1fH/LFiIZmX8oy84ts4ohgJVxK/lR/6PDsXU86nB/w/v58K8PHfbvStzF6HqjrZmQK7Pk7BLSitPoHdYbjVJzYxZ/E8jWZ6MUlH+7PAVAQkECWy9vZXzj8Tfsnjdf3mwjhhp7N+Z09mkKDAXct8a+3lefsD5olBomN5t8Q64vc41YzFIBVH2uvRjyioLOT0HLcbdMDAG09m9tFUQNvBrQO6w3gNWhuUdoD1r6taSBVwP6hvdlWqtpGMwGprWchlqppmDrVpv5TNnZqLwrIuEsxcVcuu9+u+sqPTzwGjeWooMH7fqiVq0k+eVXKImJsYohALd+ff/+Dd9iZEEkI1MLRFFk1flV1POsV21k0N+hyFiERqmxhiwXGArQKDVoldpqz9lyeYvN8fK45dWMhOF1hzOh8QQmNJ7A8JXDraUSQlxDSCpMYlfiLrYlbGNv0l4Awt3CrVmG111cx7qL65jcdDIvtnvx79zmTaPAUMCwFcMoMBYA8GWfL+kZ1vO653tr31scSj2EIAhManpt2wrVUb7NAfB0q6dp7tfcWgKjagj9iLojrpqvRuYms/4lOPyTfbuzLzwbfcuXAzCmwRj2Ju2lb3hfJja1zynkonZh4ZAKv7Ryn51ycpfaltoo3LkLz9GjrMcFW2w/U8pJe+89DAlXMGdmglpN4GvTKdq3H4+RI9CEh+Pary8lMTHW8c4dOqDQ/fO2bmWnahmZq3Ah9wIdFnXgzX1vMn79eCyihXxDPmsvrMVoqT4RWm25kn+Fvr/35YUdUs2m1KJUuv7WlXYL29k5PIMUsp1ZksnhtMPVzrlm1BqbpHiVI5S+6fcNQ+sM5cmWT/JJr08AKRNyuRgC+HHgjxwcfxClUBGN8vPJn6/7Hm82my9vtoohgKe3PY3ZcnWnUbPFTHR6NGaLGbPFzIq4FcRkxHA4VXpvt13ZdkPWdzn/sjWM+p2u7/BY88foEtzF+v6Xs2z4Mr7p9w1vdHrD0TQyt4qiTHsxFNYRBCXc+73jc24B3k7ezB8836EYqglTTg4Xhg2jcOdOANyHSE7ahdu324zLW72m2jlyflkAgOfo0XiNG0foF5/j1k/yW3TrW8mRX6Eg+EP7hJz/BGQLkYzMVXjkz0fQm/XW4zYL2tAtpBs7E3eSVpzGY80fszvnSNoR3tj7Bk+1egoXtQsNvBoQ7FoRTVFsLCbfkE+gSyC/nPqFImMROxJ2cCrrFNuubLOGWs/cP5NR9UZZw2T3JO1h2pZpuKhdMItmmvk047dhv9F+YXvrGgdHDibKI4pwt3Ci06MJdg0myDXIeu1Qt1A+7C5tnYmiSIBzAGnFadb+wVGDrQ68rhpX8krzrH3xefHWyKc7CUfWsVYLWrF33F7cNRV+A2aLmU+PfkpuaS6ualeSC5PZliCJHjeNGwWGAps5ojOiydZnWwukfhPzDTsTdvJ1v6+vWjTVaDYiInIk7QhTNk8BpIrpo+qNso7pFdrLWgesS3AXGno3pCH/ktD0fzKnVtm3PbwBzAZQ//MsH5lffInhvLTt59K1K94PTyZ//XqK9uzBYjBgKSzkQr/+Vl+gyKVLKNq3n6I9eyiNi8OcV/EZ4FHJolSOtk4U7sOHY0pLI+z771Boq7ds38nIgkhGxgElphL+78j/0ci7Edn6bJs+s2hmZ6L0pLXp0iaHgmjq5qmUmkt5dfer1rbFwxbT1KcpIIXCHks/RpfgLta5AB5Y9wACthErsZmx1nIIM/bMQESk0FgIwJA60pPe/3r9jzf3vsk7Xd+he2h3AJQKJTM6VV+BHCTfg+6h3a3bORObTOTp1k9b+1/r8Br/3V2RU2Vnws47ThCdyT5DbEYsAD1De9q8n2/ve5uPenyESqHieMZxxq8fX+08VcWQgIBFtLAjYQe7EndhFs3sSNgBSNay59s8X21pCZPFxNg/xpKrzyW9JN3aXjVHkFqpZs2oNTesVpXMDeLECunfyO6QewW6vwgKJSj+eWJItFisliEA1759cGraFKWfL+aMTIr/OkTJ0aNWMaSOCEfXogW6Fi3wfWIqotFIXLfuVlGka9XK4XWCZ38EZhFB5XjjyZBYQOa8E7j1DsetW4i1LXvJWdz7hePc0v8G3vX1IW+Zycg44OcTP/Pbmd94a99b1jZHFoHT2aeZf3K+TVtyYTKl5lK7sYvPLAakLbi/Uv/CaDHafHmXU24dKncO3p24G4DMkkyy9Fk2YwdFSrlpeoT2YMcDO6xi6Food8wEeK7NczZh20PqDGHtqLW83O5lAD47+hmLTi+6ITmKLKKFvUl7reHi18vvZyW/iIGRA/my75fMGzjP2rfp8iZaL2jNyayTPLX1qWua9/6GknPpOwfeYeuVrVYxBDDvxDxa/NKC+SfnYxEt5Opz+fzo51an6YMpBzmXc85GDIHte10ZhaBAIcgfx3cE+clweZ/0evS38FwstL0xfmS1RRRFzPn2nyG1Pt9kImfxYgyJiRTt248xKQkAt/798LznHgSFArdevQDI+Pxz8latsp7rOWaMzVyCWk3oN9+gbdKYiAW/OHwIKL2YR9L0PSTN2Eviq7vJnHeC9K+iMRcarGOyl5zFUmQib91FRLP0AJA5/ySmjBKyfzt73fd6I5H/AmVkHLD5ymab49c6vsbOB3YytcVUu7EfH/6YXYm7rMdrLjjeh49OjwZwGMkV7BJsczyq3ij+216yzJTPvf6ibdZaN40bfs5+V7mTq9M9pDtvd3mbhUMWolaq7fojPSKtX+Qm0cSHf33I5sub7cZdCwkFCTz656M8seUJPj788XXPozfpWR8vvS/3NZAitdoFtuOHAbalK97c+yY5pTkO5xhRd4Sd1cvbydtaSLVqQc3KfHz4Y5aeXcr0PdP5/vj39F/Wnw6LOji8pw+6f1Cjk7zMLUafDxtehaQjZcd5Um2yk6sAEcI6gcetT3Vgyioh+9czpLz/F7nr4ynYk4RoubYHkKx580id+TZXJk0m57ffAPB68EFCv/gChZMTAK69pb9pfWwsxmTJqd/vuWfxefhhu/mc27SmzooVWMRQ0r48hilXjyGpkOLjmZjzSsleYito9GdzMCQUULgvGXORkfRvYjBllFj7k17fS+Jre7AUVPhgll7MvaZ7vBnIW2YyMlW4mHeRuJw4m7aBkQMBeKLlE/xx8Q/ySvMYUmcIS84uAeCprU+xbPgyGng1sAqiRt6NOJN9hjc6vcH7B9/nUv4l4vPirRXJyyl3lFQpVPReKn1Ijag7grqedREQOJtzlrSiNNZelMo5TGg8gcv5l3m61dPcCARB4J7699Q4Jsw9zMbXaFfiLgZEDriu6yUXJjNq1SgMFunp8fdzvzO943S7oqC1YUfiDgqNhQS5BNE+sKJ4ZYfADoyuN5qV56VMuudyzln7nm3zrHWbs9RcikahQRAEjBYj2SXZfBX9FZOaTqKORx1c1a7W7clyVAqVjUj67OhnNmNKTCWczz1vc8760esJc7+9SedkqrD/Kzj4DVzZB5PXw2/j4XJFjh+a3Vv9uTcJ0WQh7YtjiHopIKBwVyIACo0Slw61L9ZbLoKMSUlW61BV3x+Xzp1tjj1GjcL3iSeqXVf6NzEYk6Tf89y1F9GfzHI4tjIF2xIo2JbguLOKyMv47jg+DzVG19T3qvPeLGRBJCNThaqCpU9YH+t2mUqh4tehv2K0GNEqtVZBBDBt6zTm9JhDQkECzipn5g+aj9FixEPrwabLmziYcpBPjnxCSlEKrmpXnmnzDJklmTzV6inrdskbnd4gW59Nu4B2CIJAc9/mxGbG8vPJnzmTfQaVQsUTLZ7A08nzlr0f5czqMov/bPsPBouB1RdW0yW4Cx2COuCl9apVbSSTxcSyc8t47+B7dn07EnbQP6L/Na+pPAv3sDrDbLacBEFgVtdZvNHpDXot7UW+IR+QtgQfbf6odVxli41aoSbAJYBZXWdZ23qH9bYK0Qj3COp71ue9bu9xOvs0kzdOBrATTOWEuoYyuv5o3DRushi60xBFOL5Uep0SAx+E2I9pcmtrwgHoL+RaxVBlimMzaiWIjKlF5K49jjmzwK7PqUkTm2OFTkfgW2+S+rb0++710IN25xiSC0n//Jj9Oh2IIUGrxGd8IzLnnbzqOm3W4azCUiw9YGT/fo4QWRDJyNwZiKJo3Zqa3WM2vjpfGnk3shnj5eRlfX34wcO0W9gOgPTidCZtlHwNBkQOsMkj0yu0FwdTDlr9UAZEDmBco3F21y/3Wymne2h3YjNjrTWveob2vC1iCKBLSBcOTDhAmwVSJutyZ+v+Ef3twscdMTd2Lt/GfOuw74UdLzC331y6hHQhJiMGBQqa+zWn1FyKWqF26F+Trc+2pgoYVsdxRly1Uk2/iH6siJOcZAdHDb76jVbipfYvoVaqua/BfTTzbWZtbxvQltiJsQxcPpCUohSH53YO7syUFlOu6XoyNxGzEQ7Pg9I8CG0vleOoDq9IcAuo3bT5peSsOI8poxi/aa1Quly7pROg8GAKuSvPO+wrjc/DUmxE4Ww7t2i0kLPqHIJKj6UQSk5KQkjbYhz6wxXpATwfeMCh74/XuHFoIiKw6PXomja1689ZHmfXVh0+Exrj1MALrzENKDyQjDHR9kFB19IPr1H1KNiZQMEOyfKliXDH+/4GpM6R0lyIejMWvQmF0+2RJrIgkpGpRExGDImFiTirnOkV1uuqdaG0Si1z+81l6hZb36KxDcfaHPcM68lHhyoKF1b3BV6VnqE9+Sr6K+vxiLojanXezUKtUFPXoy4X8ioy926+vJltV7bRNqBttVmiTRaTnRj6fsD3GMwGq7Pz1C1T+bLPlzy9TdoKnNtvLv/Z9h/ub3g/o+qNYvHZxQyNGkpsZiwuKhfePfguAE18mlDHs061ax5WZxgr4lbQKaiTTeqD2uDt5M3bXd522CcIAq38W5ESLwmi9fesJ1ufzb6kfWy6vMmhv1k5hsuXUXp6ovT4+1m1ZWqBUQ8b/wtHfrZt17hB5ejCtpNB7Qzt7SNHqyKKIsakQtK/jLa2lcRm4Nr52n7HQBI2lcWQ3xMtKL2QR8HOBESDBcwi+vO5OLew9Rks3HuF4iMZdvOpgtsQ8fvvWHJzMCYl4zGqemuXS5cuDtvNRUaMKbaixrltAKXxeZizpRQfftNaovLUYik2oQ50keZrF4BLuwAyfzmF/pRkSQp6rQNKd8ka694/EkuxCaW3E+69wqz9xcczcW7hd9vEEMiCSEbGhvLtsr7hfWtdJLNLSBceb/443x+XnsieavUUTX1tn7bC3MKsQiLQJZC2AbWrN1bVOtU95NqjyG40H/b4kPvW2paaeHb7swDsGbvHoSjal7zP+jrYJZifBv1EiGsIBrMBnUpHiUlyuCwXQ4BVZC48vdBqIauc7bmc4XWG17je9oHtWTps6TWLodrwcNOH2XJ5CwMjBxLmFkaYWxgt/VryZCvbDMGiwUDBjh2kzHgDzzFjyF6wAF2LFkQusq92L3MTWP4onLEPZmDQ+7DmP9LrCcugfvXbtqJFRCw1o9BJX5tF+5LJXWtrZSo5lYVr52CMGcWY0osd+sOU1x2zOe+07RaUJtwdbaQH7n3Dyf3jIoW7k0j74Ef0x+YTvmgVChc/1D5msuZvQBXQjKoICiUqnwg0ze37asKUVYIxowSnuh6UnMgECyhc1Lh0CETl5YRLh0Dyt14hf/NlFC5qtOFSjq9ysVMZr9H1yHNS4to1xKZfUAp43VPfZqzSXYtbVwfblrcYWRDJyJRhtBj589KfAAytM/Qqo23pG97XKojGNBjjcMyQOkP44tgX3FP/nlqHWAuCwIi6I1hzYQ2j6o1yGAV2q2nk3Yjjk45jNBt5autT7E/Zb+3bmbjToRWr3NfngYYP8N/2/7Xeh0apYf+4/bRa0Oq61zMoatBVxzT2aXzd819t3u33b79qmY3E51+gsKyOVPZPUgbkkiNHMKakoA4KqulUmevFUAQ5l0Dj6lgMgVSLzD1Y8imqQQwB5G+9QsHWKzWOKb2QK0VVfRWNqDfj+3BTnBpK/oell/Mp2HYF/bkcvB9oiHOrirw7xUcqEqP6PtYMQVEhmJwaeFG4OwllQDNAIHtJIoIiHZV7HEo/299rS2k+gtIJQaWh5EQWaj8XMn85idJdi/d9DWzGlpzMQuWnQ+3vTHFMOoJWRc7vUmg8gNJLEjFuPUJw61nhA+fWMxRBrcCpcc2JSZVuGrzv/2clGZUFkYxMGfuT95NTmoO3kzcdgzpe07lNfZsyvcN0vJy88NU5dgp8pNkjtAtoR0u/lg77q2N6h+m08W9zzSLtZqNWqpnVdRb9l1V8kWy7so0RdUdgES08u/1ZTBYTH3b/0JoNenS90XaiTqlQ8lSrp2y2BmvL+93er/b9vlVcrZisMTnZKoaqUrhrN14P2BfTlKkFyccg6wJEdJFETWVyE2D+MEkQVRarTe+BC1ulEPs6vUGphnr9uBqi2VKjGAp8uR1ZC05jTC0i5d0DlKUSI3PeSQJebIvSQ0vGNxW1vrIXn7UKInOBAf05KSVEwIttUfvZimvRmIZo0qNw8kBdpzeCQio2bMqvT8VzlQXDxZ2Uxv5GwFvfURwjCR51kAulcbkAOLf0w6mB5P9YfDyT7EX2le4rY84pRVArcG5n68wtqBS49bj16QhuBbIgkpEpo3y7bHDUYFSKa//TGN+4+izIIEWotQloc83zumpcubfBrQ8Brg2BLoEcn3Scv1L+4tFNj7I3aS8lphJOZJ6wOpDPOTSHUnMpUR5RNPFp4nCeJ1o+wePNHydbn41FtPDopke5nH+ZARED2J+8nwJjAYEugWQUZ/DjwB8pNZfSLqDdDatEfzPJ/3NTtX2FO3fKguh6yE+B73pJr1384MVzoFBINci2vQtHKpJzUp74s8sz0OtVSD8NB7+F/u9UO71oEcnfegWlqxr9uRz0p7OrHRvwXBtUPjp0zXwwphZZxVA5af874tCaYi4wYM43kP6FFMWlDnW1E0OFe/aS8NhjOLWfijqkLU4txtrNow7IxfeJgZSe9UTUD8CpeVuKYw9gSiumcE9F0eDMn07g92RLtBHuFJSF818N59b+1+0k/k9EFkQyMki1xbYnSIUOh0QNuc2r+efRPrA9wS7BJBclMzdmLj+e+NHat/rCakBybq6u1AVIlqLyRJPPtH6G3878xovtXrT6/pgsJgoNhbctyu56EEWRvDVSXiqnFi1watyY3CVL8LjnHvJWrKBw2zZON2qM77Qn8Xvmmdu82n8QJyr5khVlQEo0hLSB3yfDpd2Oz+nxMmhcILQdhP7geEwZ+lNZDi1C2gZeOLfyw5xbiqXYhFvPUJRukijXNfclf4tjK5IjQVVyKov8Steo6jAtWiwkPCY5d5tSolGHOPY79H6wL0qdBudWFZZnTZgbhisFGC7n24wt3J2IQheJMcE+LL8czxF1yV1TVvfsGnIf/RuQBZGMDLAtYRslphLC3MJo7tv8di/nH4cgCPQJ78PC0wttxFBlrkVoDogcYJf4UaVQ/aPEEIA+JobS06cRnJwI/24uCldXPO+/D6cmTSShZJL8NTK//gaPESPQREb+7WuaCwowJqfg1LDB1Qf/U9CXfbHv+AAOfG3ff2yhlFW6OjHUaBg4uTvuc0BxjH3kFoB7v3CrI3FV1AEueA6vQ+7aizi39kdb35OcpRUJQVEI+D3WjNL4fPI3X7YLsdc1t936LYmOtr42pcVaX1sK09AfW4Cm0TB8Hh1iZ1UCye/IcEUSPYKTClEv/Z6VXi6g6LDkr+TUyBu3PmFk/XIKhYsaU1oxSncNLp2CEE0WEEET6lbNO/TvRBZEMnc153POY7QYWXNeeoofWmdojVYMmerpF9HPGg1WjrPKmWJTMW382xDq9u/0O6iJ3NWSdcxtQH+Unp4A1nwvHiNHkLd8hXVswZYt+Dx29XDvmijcu5ek51/Akp9PxMIFOLdr97fmuyMwlsDXnSHfwTbPgPdg0+tw+EfbbbKgVtBhCtTpBdG/Qsfa54OylJjsor4AXLsGowmrWSC4dg3BtVK0lFMDL1JnH0I0WNA19UFbxxOlpxP5my/bnOf7SDNUXk4YEpPIXbqUnCVLUOgqolwD/vsinmM6Y0gpJHveHsxZ5/Do74ZbF8eRWdoGXlBmrXLtHIR733CS3zmApcBA4T4pc7VLuwC04e4Ez+gEQOmVfJSuGgSF8K/1EboasiCSuSMoNhbzTcw39I/oTwu/FrU+71LeJWIzY+kc1Pma63pl67MZvWa0TdvQqDvLcfmfRCu/VjbH73d7n0Oph1h5fmW1kXf/ZkSDgYL1GwDwGGGfBybg1VcRDUby10qZsAs2b8F9+HBUXl4Immv3jdKfOkXCoxWCKnvRon+HIDq30bEYqtsHWo6VBBGAKBUMZcC70OU/FeN6vnxNlyuOzQCTiMJZhcJVja6JDx6Doq5+ogOUrho8hkRRsD0Rt95SpJbK2wmlt5M1l497/wicGnghiiKXx43DlCFZpyxl1eVDvvgc9/5S4IJTlCfBs6YTPGt6jdfVhLqhdNdgLjDg3MoPQaXAqZE3JTEV9+bUyNavqTrL192ELIhk7gh+PfMrP5/8mQMpB/h9+O81jhVFkc+Ofoabxo0fj/9IgVEyDW+9byv+zv41nluZ8hD7clzVrkR6RF7z2mUklAolL7d7mSVnl/BV36+I9Iikb3hfxjQYc9dtQxoSE7nQT/oSU/n54dK5k90YpZsbIXNm4//SS5zv2ZOSmBjO9+yF5wMPEPT2zFpdJ/Pbb8n49DOcWrZAwNayWXzwL0SzGUF59bIqdzSxVT4PgttI1ecbjwBnb2gwSBJN5TS3zZF1LYgWkaIDUqJNt15hN8RS4topGNdOtlFwvo80I2v+SZSeWly7SH3648etYsjm/G7drvmagkLA9/HmUsLEAClhoq6pjySIAOdW/ggqubZ7VWRBJHPbSShIYOEpaavlTPYZUotSCXSp3pnvaPpRh34qmy9vZkLjCbW+btWq8w83s6/yLHNtTGw6kYlNJ1qPndXO12TxuxOQMhAnow4JrtX2qf7sWQwXL+I+WCoLkvHFl2R+VZFCwH3YsBpFiTrAH13LlpTESGHZuUuWgMVM4KxZNV7fUlRExqefSWuIibXrN2dnU3zkCC4dOkjjzRbOH03HUGLGO8iFoHoed/72cHE2xJVF6fk2hPCOMPB90FbauhozD5ZOhPObwa8RuF2bI7BoETFlFCOaxYq6XSoB57a1K91RWywlJVj0elReXqh9dQS+2A5zfj7m/GwUOn/yy6yJlVGHhNhsnV0LVX2LnBp6IagViEbLDb+3fwuyIJK5rSQUJDBkha2z7fPbn+enQT9Vmym6avHVcnYm7Ky1ILqSf4XYDOlL5P1u72O0GBlZ99YXc5S5s7AUFZH5zTdk/fAjQe+9h+e991Q7VjQYSH3/fXIXSwV+lR4eOLVoaSOGADxG1JxJG8ClWzerIALI/X0Zzu3bk/6/T/AYPQr/556zOyd/82aHczU6dZKU114nb9Uqkl9+BVNaGu5Dh5LU8wkOra+Iaur3cBMadqydeMhMLODoxst0GlUXd9/r+4K+Lk6vAYsRAprBk3sdj9E4w5ifYN8X0Oaha75EwfYEO58eXROfGxZubjEYEJRKEh6fgv7UKaJWr0ITFoZosXD5wYcwJiZSZ91a8v+ULNb+r7yCOjiI9P99QsjHc27IGgAUWhU+k5tiKTaiCXG9YfP+m7itNrNdu3YxfPhwgoOlJ7FVq1bZ9IuiyJtvvklQUBA6nY5+/foRF1dRbO7SpUs8+uijREVFodPpqFu3Lm+99RYGg8FmntjYWLp3746TkxNhYWHMnj37VtyeTC1YGbfSru1E1gn+d/h/1uPdibv58tiXXMy9iMFssNvqmtFxBgCH0g4RlxNH36V9aT6/OTEZMTgiPi+eh/+UrEFdg7syvO5w7ql/T60qtsv8ezAXFGBITEQ0GgEpzPnS+Alk/SBZH1Nef53SCxdszhFFEVEUMefnE3/vGKsYArjyyKNkzbWt1xb03rs4Nb56lmyvcWPRVhmX/Mp/MaWlkfXtXPSnTpG9aBGnGzXm8kMTyVu7lpRXJT8SpVdFseHAt99GUChwGyBF6JnSpIiivD/W24ghgC3zThFdTZh4Vf74LBr/M1lc/vwYWYvPUHLK3un4pnC8LLy++VV80Jzcoc/r4Bl+TdNnLT5jJ4aAv71VJhoMZP38MyUxMcR16cqZps0oPnwYS3ExF/oPIPG55ynctYvSc+ektmHDMaWkoHBxwWvCeNwHDaLe5k3oWl5bEter4VTXE+fm1+ZreTdxWy1ERUVFtGzZkkceeYR77rF/Eps9ezaff/458+fPJyoqijfeeIOBAwdy6tQpnJycOHPmDBaLhblz51KvXj1OnDjB448/TlFRER9//DEA+fn5DBgwgH79+vHtt99y/PhxHnnkETw9PZkyRa5EfTsRRZHNlyuecl/t8Cof/vUhAEvOLuH1jq+TUpTCtK3TAKla+pyec8g3VOTW8Nf5M6bBGBadWUR8XjwPrHsAo0X6gnvsz8d4pNkjfB0jheluu28bRcYiRqyqKC1xp2V/lrk1iKLI5YcmUnrmDAANjxxGf+YMpWfP2oy7OHQYjU6fsm4tpbw+g7wVK+zmKyfr+4r8NhG//Ypz69a1Wo/K15c6K1dgLizi8rhxlFZ68AOIv6ciMWfxoUMUHzpkPY5a9jvqkBBEiwVBIT3junS1LdiZ7eW4hMLeZecx6M20HxqJIAiUXryIKSMT5w7tid2WiHeQC77hrviUmvB1VoHBTEl0BiXRGQTP7HzDC3EWx2RQsCsRt85+OB99UMpGDVKG6RuIpdhI2hfHMOeU2vX5P9sGTZDL35o/89u5ZH7tID1AGQUbN1KwscLvSSyWkke69u2DQmtfF0zm1nBbBdHgwYMZXLbvXhVRFPn000+ZMWMGI0dKWxm//PILAQEBrFq1irFjxzJo0CAGDaqoY1SnTh3Onj3LN998YxVEixYtwmAw8NNPP6HRaGjatCnR0dF88sknsiC6zZzKPsWl/EuoFCpWjVxFuFs4sRmxrI9fD0j+RLuTbPOKfHxI+n+d3HQygyIH4eXkhVKhpE9YH37M+9EqhgD0Zr1VDAGMXD2SAoNtQrK+4X1v1u3J3MGUREdbxRBA2pw5Ntaeypxp7Di7dk3UWbcWbb1613ye0tWFyCWLOdumdsV/BZ0OdYgUel0uhgAUWi1OTZqgP3UKgDR/KdosJHk3o9a8xcJXtpFX9lxxaF08YQ09EFb/TPSf8fhkncT93f9jz5oKK1BXV3vrafLM/QS91QmFSkne5suYMkvwHFYHlbdTre9XFEXyNsRT9FcqXqPrkf2bJEizlxViULbCSSGi8PAg55cs3Ae5oWtoGxllSClCoVOh8qy9iDAXGEj932FEvdmuz61P2DWJoby168hZuJDg//0PTWgIhsQkBLWa3GX2RYhrg8fwq2+vytw87lg38/j4eFJTU+nXr6LOjIeHBx07dmT//v3VnpeXl4e3d8Ufzf79++nRoweaSmGsAwcO5OzZs+Tk5Dico7S0lPz8fJsfmRtPuS9Qv/B+RLhHIAgCH/X4iD5hfQDYemWrneNzWrG0BTCszjCa+ja1ZjHuF3H1ekRVxdC0VtOuWpRT5t9JefbociqLIb/nnsV/xv/h1O5xBG3NeWe8xo+n8ZnTBJc9gJVzPWKoHIWzM43PnCbi118J+fyzGseG/1h9xuWQ//sE9yFDiFi7jkz/VgD4px3GkJhI0z9nIFhM1rG7vtzF8vOtuFB3NNEtn+bC7oqkgU4C+CgdO1+ve2UPxz89SuGuRPSnsshaeOoa7hSMSYUU7kpC1JutYqicQvO9ZBrfJT3zZYwpRXaZow0pRaR/cYyM72MRLVXqZdRA5vyTNmJI4arG9/HmeI6si3ufii230osXyfrxR+uWalVEUST55ZcpiYkhffZsTBkZXOjXj/M9e2JKT7cZ6zVhAoHvzCLit19ttsEU7u74vyylBXDt2ROX64gok7lx3LGCKDU1FYCAAFtv+ICAAGtfVc6fP88XX3zB1KlTbeZxNEfla1Tlgw8+wMPDw/oTFhbmcJzM9WG2mEkrSmNjvGQyrprBuG+EZLWZGzuX+Lx4tEot/+tZ4VNUz7MeDbxss/A29Wlqfd3IuxFTWlRY/xwV//y237c82fLJv38zMnc0+tOnyd+4EWOa9AUliiLZCxaS+9tiADzG2NeI83nsMUpOuKAObY+2mX2dMU3Dvrj0foLwXxcR+OYb0jzDhuL9sOSXFjD91Ruyduc2rXEfMICoFcutbZHLluHcsSNKDw/q7dyJc5vqa+Nlmz3Z5zeWmBNgUurQlObimXeB1Jlv41SaQ9f9r1vHZpZW5KAxaD2Jz6nwSwrWKBAEAYW/DrGOB2lGi7UvUK3ALbPYemxMLiJnhe12X00UxzrOCO0Iw5UCTHnSFlfxsXTSPzsKFhFzlp70L49hMUgipzQ+nvMDxpL6wSd2c5gySzAmFlqP3fuFE/R6R5zqeuLaOdgmFD1l+mukz/mYnN9tw/5Fo5GcxYspqOTUXhIdbSeyy/GePJnAN2bgdd99OLduTeSSxShcJCuU5+jR+Dz6CI3PnCZs7rd3ftTfv5x/TZRZUlISgwYN4r777uPxxx//W3NNnz6dF154wXqcn58vi6IbyIy9M6yWH3eNO91CbJ+Keob2tDnuGtyV3mG9cdO4UWAocJhNWhAEXu3wKnNj5jKz80wsooXvYr8D4Jt+37D4zGKWxy3HRe3CqpGragzrl/l3YExKIn605HuiiYggYvFvJD7xpDWaS+HsTNDMmeQtqxAcYd/NxZRV4VeiDm+HUnsGXHsimktR+6mx6OuUnR9pcz2/557FrU9vdDc4GaK2cWO8xo8HlRKnpk0I//47RJMJhXPN1s11X8ZQWmwi/bJkGY0MUyDsFynaswcAjbGQbntfYU/XmoNMQtXS35p+/1JUbpkox8/k0OoLtHdREaKxf6Yu+isVtz5hqDxr3joTLaI1L47N+VvfQtftZRRa+0iokhOZmNKLKTpo+zBrTC4idc4eSg79gaDS4dTmKfTnzyIaDAgaDdm/rqTo4Flculb4D7r1CsO9X4TDtRlTUqy/J4Vbt+E9fjyiyYQpPZ3cZcvt/INM6emkz7G1EgbOfAtNZBS6NvZ+ZOHz51OwZTO+TzxRzbsjczu4YwVRYKD0hZWWlkZQUJC1PS0tjVatWtmMTU5Opnfv3nTp0oXvvvvObp60skiLynNUvkZVtFotWtmx7aaQWZJpsw3WK6wXaqVteKuH1oNeYb2s1dJH1huJWqnmuTbPsfXKVu6t77jy+4TGE6xh9xbRwpgGY9AoNDT0asjMLjN5pf0rmEUzbpq7qz7P3UreH+utrw2XL3Pp/gcwJiRY21x79URQqQh44y2y5q0i+IMXcWnfnrzKUUeiEk3TJzBllABg0Vd05a46T4GnFu/7GqDy0aHQanFu3/6G34cgCFZLFAAazVUzWZcUGCgtNtm0NRnciPylFcdKLy80VdwGfEJcyEoqAiA4eQ9BBRfx6vgoomjBcG4npaX5hP2nhL1GkRKLiE5R5mxusGABq0DSn8nBtVMQuStWUrB1Kz6PPmJnzTJcycecZ0A0llC860Oce07HlHEaS0EKRRteBEHAbaQUtSea8hFU7hT9lYoprRhHWAoUaBtV+OCo/BqSv/0guYt/QnAZisK1K8VHJb8or/sa4OIgF4+luJisefMw5+Ra24r/+gtjWhqZX35FbhVrkSPCf/oRU1aWlH+qGouPrllTdM2aOuyTuX3csYIoKiqKwMBAtm7dahVA+fn5HDx4kCefrNjqSEpKonfv3rRt25Z58+ahUNg+sXTu3JnXX38do9GIWi198W7evJmGDRviVSlcVebWsCHeNvnY6HqjHY57v9v7bLuyDbVCTe+w3gDc3/B+7m9ov4XhCIWg4K3Ob9m0yf5Cdxf569fbHFcWQ0pvb/xflba2BF1bdO1CyFmuRxNeZGe1KBdDVTHnlmLOLSV33UV8J91ZX27nj9j6sAgChHRuRGVvSL9nnyV15kxaxH7NpVYPMuC5rjh5ubLwDclHMzDtL/z86wNgzjiLWCqdLR7aRbf7BmE+ngGpknhKNlpINYl4qQScFQL601kU71tMVtkDauHWrTSKjbERcuUFVE0px7AUpFC44SWwmHDyMqLPUYMoUvLXXJS+DTBc2Ipr/3dtxJA6yAXvcY3IXnoSY2IlpVqJ7AVbMJxLxKWPrfjRNfNxOD7944/J+fU3mzbRaOR8z14OxwMEzJhB2rvvAuAxciQuXbpUO1bmzua2+hAVFhYSHR1NdFlV3/j4eKKjo7ly5QqCIPDcc8/x7rvvsmbNGo4fP87EiRMJDg5m1KhRgCSGevXqRXh4OB9//DEZGRmkpqba+AaNHz8ejUbDo48+ysmTJ1myZAmfffaZzZaYzI3nUt4l9ifvJ6skC4toQRRFPj/6ObMPSeb5CY0nMLf/XNoFOt5ecNO4MbLeSIbUGSLvq8tcM6UXLkhRZCoVvs/8x6bP4557qL93D2p/f0SjhaK/Kj4vshadxpRZAioFwW91BoX9755QJdTckFBgN+Z2E1dW0dzd1wmFSmDY05Ijb+BbbwIQPPsj3AYOkN6f7JPc+3gEAQ388PDT0WFEFC27+RLWszXqiK4AmJIqwvwLtm6hZd8wIvpXbDdFZS7HJMLBQskqpT+bRdaP823WVHzkiPV11rz5FO6WcjwZE8vmNhtAtOAWVkLgAOlh1ZR8hNLY3xCL0jHnxFvPV7iW4vd4c9T+zhhO/Ywx+ajD90Ed2g+XPm/atBmTjyJo7KPmRLPZTgyV53OqDo+RI/CaMJ7gOXMInDnT1pIn84/jtlqIDh8+TO/eva3H5SJl0qRJ/Pzzz7zyyisUFRUxZcoUcnNz6datGxs3bsTJSdqb3rx5M+fPn+f8+fOEhtom0hJFKerAw8ODTZs28dRTT9G2bVt8fX1588035ZD7m0iJqYThq2oOH32ixRN4OnnemgXJ3FVYiorI+u57AFy7dsVz9GgyP/8CkKLCfJ95mdQPD+HUyAunBrZh3KZ0yRqka+SFQqdC7e+MscwKovJ3Rh3kgs+4RqR+cgRTumStsBQaMWYU25VKuF0UZOtJOZ8HAox+sS0unhrrQ4Xn2LG4DxuG0k3aNg6ZMxtTZha6Sm4I7TuYMWUrSD0xAIUrIFqI/O0TLIU5XBgwEH1MLMa0dJwa+2HOPYrhYgzqK/tp8uhwTl1Ql22lKXAb/gWFG19B1OcCULhjJy6dO2NML6DwoAqlpzOW4mzMGadx9i+lOF1yU3APK0Fz30RM9Yw2vjqm5KMovaQiq/nL3iDov7sQTSZKjh7FnLsNPRD28884t2mHITmTzG/P2bwvpsS1GJMSMGWcpjRuME4NGlBy/DiX7iuzOjsor+I1fjwFmzbZtfs9/zzqkBDc+vRGEAQ8hg+7nv8qmTuM2yqIevXqZRUujhAEgVmzZjFr1iyH/ZMnT2by5MlXvU6LFi3YvXv3VcfJ3Bh2JuyssT/ENUQWQzI3jcRnn7M6DrsPG4o6KAjXnj0pPnwY70kTyd9yBXNeKUUHUyk5nQ2AS8dAio+mI5ZFUOlaStl8PUfUJeOHWDwGRuLWsyKwwufBxuRvvSL5weSUoj+Tc9sFkUFv4sSuJGK2SFuDwfU8cfWy9YUUBMEqhgBr/TUrpQXwbVeKDOMBySFd7V6EytsNvN1watkCfUwshTt24NqrJ8W7vgOL9J4F/vgcmU0eJs+lo9W3SFO/D95j25Dy+gwKd+wgYPqrZHx7FGVZRmnj5T2onQ0EtMkj/k8/dL4GNG5maDoa7yY69GfO4Nq7F4aL8eT8thJ1vYGYM88g6nMp3LsXw4ULmHNzAQj96ktcO3UEwCkyAKgkiJQCYV+/yZXHHsWUUkrhjp04NWhA0rPPVYwxS1FqqoAATGlpBH34Ac5tK/yeAmfORNeyBUX79uE9aRKC6o71OJG5TuT/UZkbTnlixer4ss+Xt2glMv9ERKOZjB9PYLiUj9e99XFpX/uIQGNSklUMCVp3nNtLEYyhX3xO0bE0Si8L6M9VOBJb8qUyP85tA7DozZTEZCBolDiVJQDU1vEg5N1uVCkkj9rfGZ9xjSjYk0Teuovk/XERQSlYK5ffDvYsjeP0vhTrcYMO11HA8/Q6MOkpMVYIAU/PFYCUGsOtd2+rILIUF1vFUDl+GTFcCG5PoLosY3bPe3Eb2ICUt2ZiuHwZ/dlLWIorPDWMl/fgGVaKk6eJukPSUWotUH8AuAWgBMK+lurClcTEkD1vnuRsXUbiExW+pF4PPohbX9skqyqvHEw5XiAYCHiuM0p3ZzxHjyI1NpbCnTvxnjAeY3Ky3VsQPm8emvAwq+CJ/H0pprQ03Mpy4tWmFIvMPxNZEMncUPJK89iTJH0hLRi8AIBTWac4nnmcRt6NmNhkouwTJFMthqRC0r84Zj3OWR6Hc2t/m/ww1VF6MZ6LQ8pyWik1uA5+n7RPTwCSsCm9mFftuZowN1zaB1ASm4FLuwAUlXxMBAd+ROU4NfImb91FAHLXXEBbxwN14N8r+3A9mI0WGzEEULe1/zXPI8YupdB0LyYxEjATqH0MVVYulP4faN1w7dWLjE8/o2j/fgwJUqJE74aFZJ+VQuR9sk9yurSU/UBnVxXGxCJiNqXg06oV+gtZZM6rcGw3xa9EMGXj3UDKC6RxM0OnadDzFbt1ObVoQcAbM1D5+KD08ubKpEk2/R4j7Qsz+z83mNxVh/EY3Balu2S9c+0ppfQoOXbMYd4g9+HD0daJsmnTNW8OzZvX6v2T+WcjCyKZG8rmy5sxWozU96pPq7LsuOX/yshcjaxFp+3a9Odz0Ya7kfFdLOoQN7zva2A3JuPrr61+QgBek1/BlFUR0VRVDGnC3KzO0B5DohAEAad6XgS91hGFc+2rnKt9daj8dVbfo7RPj6JwUePc0g/PEXVrPc/fJT420+a47aAInFyvsVp79K8UnPMm3zQZACfFUVRCBliAC9ugOAutWzCqoCBMKSkYzl8AhYBP40KMRUoKkpxQmUvxyT5Jul9rCs0FuCrdSN2RhHeH3ug8K5I/muN/I6LBIiz1BJRqER7dDFp38G/kcGmCIOA9QUqpIZptS27o2rTByUEIu0KrxvuBzjZt6uBgtA0aUHruHKlvS64YPo8/jv+Lkv9qTS4cMv9+7thM1TL/PAxmA2suSE9dQ6PkoqnXgmgyYS4svPrAOwGzCc5uAIPjfDDXStHRNEpOZ2EpNWPOtg+fzvr5JHl/XsKYWkzxkTTMebYFOY1JSTZiCEDpXXMYvFvfcEI/7E7Ie11x7S7VAiP9NMof2yIcqb4chiP8HmuOUClBoaXISOG+ZCzFjks+3AxO7ZW2ftoNieSpb/vQadQ1ijGLGTa/aRVDALrIMosNwNKJsO55hN8ewK1PReJUl4ASVE4WQrvl0PgByULlmxkLwCWDZJVp4azEVNQIhWvFFp6LZjeCSoHy0VUw9jcI61CtGKqKoFTiMVpK1+HauzcR83++Jquza5/eNsfuQysy5cvW67sbWRDJ3BAyijNou7Atx9Kl7Q65inztsJSUYCkpIeX1GcR16Yr+7Lmrn3Q7KUyHpQ/Bb2Nhy8y/PZ0hqZCcpefImn+K4mMVCVR9JjbB99Fm1uPKmYlLTmXZzJFfqYQCgPfj71EaJ1l/nNsF4NREyjmjcFYgaAVUfjqc6nkCICgVFV+C61+G3Cuw/iU48nOt70HpriXgGfsSGiUnsxyMvvHkZ5WQUOYc3qhz0FVGV8PlvRgLKlmUFKB76HnJn6cK7ikVPoDu4WU5mu7/BYCwXllEaA4DFpIrlfhwyquw6hTt+hCPoETwawR1ekEj29I9tSFwxusEvDGD4I8+RFBfmyWssiO5tn49nBrVTojJ/PuRt8xk/jZbL2/luR3PWY/dNe53bWkMc34++jNn0EREog6o2YfDUlrKhcFDMFXKm5X9008Ef/ShZLoXRZsK5reV0kJQKGHhPZB6XGr7ay741IOO15fCwpxXSt7GitwyuaukvDRuvULRNfFBNFscnlewJ4nS87loItzJ3/wXBWu+RuEeirb5fbj17UhJrJRAUBXgjPeYsu01swk+b4VYmAuT9yMUp8KyR6FeX9DnQoepkFzhu8TaZyGqB3jXqdW9qHx1+E1pTsZ3x61txcczr8kh/Ho591caiBDS0AsPP931TXJiOSUWKeeQOswN38lNUehUEGGfZFDna0DracRsEHAL1UOd3tBkJPSajuuOD3Cd+ATq3wRKHBjIEosK8FOeR+1sgRFf2A+oJQoXF+sW2rWibVCx5epaqXi4jIwsiGT+NpXFEMB/O/z39izkDiDppZco2iWleKizYT3aKMlB02IwkLd8Oa59+yIIAio/Pwq377ARQwCFe/Zgzs8n6/sfyJ4/X6qO3fQ2Z0EuycXw+X0IRUmoFUm2fRtehjo9wa/hNU0pWkTSv4nBnFtq16drLoW8C+f/xM/jazLybJ1szVl6SrL0ZRYYP7TN7kPp3wSF1t0qhgCc6lfKRH9pF+QlSMFiZ1ZC2km4sk/6Adjn4Mv5zHro8nSt70kT5YHHsDqIRgv5f16i9FwOGT8ex3dyM4RqqsXfCMqzUl9XVBmA2Qin1lBilhIYurYPROlSZnVRaeHx7bDueXAPhrPrERQQ1T8DEVD0fB76lCUj7PWq9AN0E5PZvuAMiW75hBZIvkOiKHLK5ISh6WeEv9gVtdvtKaEjCAIRvy6i4M9N+Mr56GQqIQsimb9FapHtF/onvT6hX7jtU5elxISgUSAo7xBrx03CmJZmFUMAucuW4TF0KFk//4w+9jiGS5fg7YqcWtom9uG75qwsznXoaD3O/PwL/J5/Dm3dute8NXCjMC2YSnrOmwgUEax9CEEw2A74qgNM3QVBUjZkEg6BVwS4Vm8hK43PcyiGANQJi+CKGf6cjhbwUqvIMb6A+dwPKCNHgMZ2XlVQGwSVbW0vQa3ArWelZK1Hf6l4fXot5CU6XlidXnBxh/T6zB/XJIgEQcCtm+SPlP/nJQBK43Ipjc/Fqd6NLRNkNllY+3k0SedyAVAoBOqU5U6q1fn5BgStEoVWCRd3YipSYxTrgQBOTWyTVRLSBqaW5RZLOwnfdEFQgtD6Ieg9Q7IcVqFxlyDcfXX4+5s4/t4rHCh8qFKvhu9ePsTE97vg5l1zAdibhXObNna11WRkZEEkc91klmQyY+8MALy0XswbNI+6nrbOnMbUItK+jEbX1Aefcf/evXr9qVPE32NbdLZoz16y5/1sl6ulnNJTUkSVz5QpaMLDyPjsc0wZtnW0CnfupHDnTptImBuNxWAmZ9k5VL46jEmFWEpMeI9rhNJVjSUxjpIr0jaMiAt6S0t0Yx6HhoPhq45QUJbHZe/nMOZHSUz8MhLCOsKj9hl+AfTncsj86YT12K1nKK49Qincl4wuoABhha1FyEW5jZLzMWSc0qLVNEQTaSuIqoohgJB3usKuOZLjd9dnJHFTTuJf1b8ZPV+FEV/Cp80g4QAUZoBr7YWG9Z56h1GwXQoxL47OQFvHEyE1GoqyoP7f36a5fDzLKoYAvENcah1VZswoJu1/UhkNt9YWPLQrrNtl2joeKF1rKBwb0BSG/R+kxMLg2aB0/BUiCAKhDSUR2HbOT7T9tDmnUhqyPf8p65idv561lhSRkbkTkAWRzDVzJvsMp7NO88mRT8gtzQXg6dZP24mhwgPJVr+QkpgMzCPqVpji/0UUHz7M5QcrnoB9pk4l6/vvKT179qrnahs2xP+F5wGp4GjitKccjsv6/nv8nn/upvgUFR9Jo6RK2Hb2b2dQOKvRn8kGHq1Yh3EIwXWGIGhcESathS/bSh1X9kvCb/f/pOOEg5guxoFHKCrjecQLu6HTFBCUNmLIc1Q9XDoEIigEPPpHwI6P7NaXHedMxjEp47Lh1CoEtTNY9KgC2yKoKywMSlUOZpMXXvfUh4yzsE0quElJjlQny78JOHlIawXp2CJlRWb3x+AZIQk5hUKydqXEwNn10HZS1SVdFfe+4ZgySyg5nknx4TTMOcX4JfWSOp19SPSegPuo13D3vT6fnzMHbHMONS2PlLsKJWezyZp30npccEyBk+YwJebHAdA18736JO0eqf1CQaos23AIjXPnYlG7sDNrMgAJp7IRLWKNeZ5kZG4lsiCSuSqxGbEoFUqa+jSl0FDIfWvvsxvTP6K/zbFoEa1iqBz9mWxc2l6nn8MtwpicTOb33+P7+OOog6+edVh/6pSNGALwfnAC+uOxFO3b7/AcTb26Ug4XwLV3L2u7W58+1NuxHUN8PMbkZLIXLJQKlJZf6/hxdC1v/BO1JHpsMVyprmBpe5Lfi8atVxgeg+rB86fgsxaQnwSzKraFLKKO9O/OIaqS8PX8kozMZ2HtfnQeZ4EKf6NyMQTApjdg3+c2VzOO/J20cc9aj0VDAfpDc8uO5qOu1x+nZvehII8A5eOYXJqgFibCT29XTHJknvRv8zGgcqoQRIM+kLbIABoPA2cfSQwBNB4hCaJTq2oWRGfWS2OC24CxCLq9AIKAoFLgPciFpOOS0Cy9UECmYgY+6g9JyfVn9cXeuMb9xcQPeyCc3yL56kT1kObMOAvedau1vpQUGLh8XIpgC2/qQ51WvjTpWvPvau66i+hPZ2HKsk9rkGEoE7EC6JrWQhBdD20nIxz/nWadmmE21WfP73FYLCJL3juEq5eWQVOboVIrKcwpJT+zGP9Id1Rq+604GZmbiSyIZGokvTidCeulaI49Y/ewK3GX3ZiRdUfi5SR9GVqKjeSsOo/SU2s3LnfNBQr3JOF1T300YbfHofJqJD7/PPqYWEoOHyZy+XIEQSB+zH1Wa0/wxx/jMWwooihS/Nchu4y5Qe+/j8rPD7f+/a2CKHDmTLzGPoBoMFC0fz8uXbqQ8fXXFO2VaiJVRh0YiDpQikxS+vjYlCco2LLlhgsiS4kJ/fncq47LVwo4GS1oysRLwY4Edp3LwWQUqev6Ms3zPqByChe9pT0W3MEEGZnTrO0leRViyK/VAYQ/lkGj4ZKfSmUxpPOG4FbkR1dy4haAKnnz3Ps1RGHJwDlnHgpBj6b0KPzhuPI5zcZIwmPHR+ARCpE9KvoCq2Qibjoatr0DF3dCcbZ0nqAAdSWLjsUCi8dJr2OXSP+Gd7ZGZgln1+CqjKfQPKrsPelEkXkQF0qkdAKFeWYyjxzCa+04tuQ+h08vX9pHRcOqJyVh1e8t66XOHEhh9+Jz9Hu4CVdOZmOxiPhHuDH8P1f/fbDk5lC4J8mu3UlxAL2lk/VYW88TpXsN22V/h4Am8F8porAlkJFQwNkDqWQlFZKVVMjlE1noXNWs/J8U6de0Rwi9xl+bo76MzN/l3+3lKnNVzBYznxz5hNXnV1uPT2aetGZs3Ri/0Tq22+JufBX9lc35H3T/gLe7SE/jpVfySZ51gJLYTAp3SR/ASm8nvMdLvkNiqRljShHZS66+leQQiwVMhquPu04sej36GCmpXGncec517MSZ5i1str6SX3oJ0WAg++f5dmIo9Msv8LynLGFcn76gUiGo1bj1l3xGBI0G1549EdRq/J99lqilS1B5Ve9s69arF1GrVxH03nsAFGzeck2ZdC16E6lzDpFRaYuqKiWnssAsonBWoQ51xWN4HZzbVIjVs3ozscVm9mYbyDbbXrvobC7JcbnsPteBMyV9Kjq6v0iJuXK4tv02qZ/mJbRn3pXy/Sy6Fw5UVDWn/gDpy/OhleRvqPj9q7NqBQ0OVfj/eIy5F7+npuD7wj04D7bPlwOARiopQVgnydHbLRD+cwQe2VhhDXKET11JJIlmiFkMX3aAb7tLEVnlJB22P6+yr9KJ5Xiofkan2GFtyjU9QZS2GxFliRzP//Yz8foOXCjtwqFNKWQv3Uqu8WHMu3+QtvOA+JgMtv58GoPezPpvjnOi7G+rtjmHShbb/s0qyCJAMxUvtW1NwVuRIqCcum1s/cDiYzI5sPqi9fjkriRMRnPV02RkbiqyheguJq80j0f/fJSzOdIXfr+Ifsw/OZ9vYr7hlfavkKPP4fvj39uck1QofRivHLESd607/s7SB5s+LofMH+2/eL1G10NblgSvHFNmCaLRjHCtJvG1/4ETK2HKDvBrANG/gXcUhHe66qm1oXDHDptjsaTE4biigwdJ/8jW16XO+j/Q1qnIWaMO8Cf8xx8RFAIqH5/rXpNTw4aoQ0JInTkTw6VLGC5eRFv36lmIKzvOmrL0GNOLUfvbVmM3JBaQ87uUCNK1SzDu/SLgxAri1mxFJ4zFLIqc11swlY2/YrBYi3YCNHdWkm8WOVFiJq60O42dt8Gw/8PS7CH02/bWuD6NcMa2YfcnAIhBrUjY4kLRO5Ui8BQK6u/cgcpPcm4O+uADjAlX8H366Qqfqk7TILQD/FRJGHV+GgyFkuiqvO1VWyfppqOlnEt/Tq9ou7QH6pZlOo5eZH/O/i+h8XApwi75KIIAXuovMBv8MYhNrMNaOSu5bLBwrqQH2eoIAOpolBRbJgNgsDTFb/5oDP3nsP4be+tOoFogzMn+70e0iIgmS0UttpJcSq7YRnJ5qb+W0if4NiSocxYpy70Q1GqcGt7YSLiaCG9sG8kWdyjNbsyVk9nUaeX4/6qk0ICTs1r2P5K5ociC6C7m9T2vW8UQwK7EXXwT8w0Asw/Nrva8+l71qedVz3pszit1KIYAtHU9EQQBl/aBFB2qCNFPemMfQW90qp2TtaEYfuwPaWXX2P0/aeui3DdkRgY4iDS6FrJ/+YW09z8AJEuOaLC1ROlataIkOhqAhMcrcpeoQ0OJWrUKpat9QU+Xjh3+1prKUbq64tylM0U7d1GweUuNgqjwQAq5q87btZecykLt74woihTuTqLkZBaGyxU5e3Qt/Ujb+Bs525dwtqQnWsFMsUWklesyUoMmkxhXSIpR5JKvMwUJBTTXKfFQCngoBdwUcLC0FcaH/kRdtxP645mIZiVKTzVifjoWixc6n0s45/9ClvFVvNWfYVchQZSsASUNXqTo/2zzWOlatLCKIQDP0aPsb1yhhPCO0P8dOPANjF0EQa3AXAqtJ0pbctdKk1GwdZZt2+k1kiAylsCJFVKbRxiUFkgJHgF+GgjtH5Ne1+mFIjse/9xXSNSvs5nKTWGiwOJPYan0UFFHWyE2DWIjks42Qn/mB7yUQ8g3i5TbSzQCdHBRUbjqPC51PGyEbv7my9botsBX2qO4sBG9Rbp3N+WvCIIBJ0WZla3JCJRtRxEQWgSCgEJ7674OlGoFAx9vRsqFXM4eSKW0WJLdbt5ORDb34fjOJOIOpzkURJdPZrHuixgARjzbirAq4kpG5nqRt8z+5cTlxDH/5HxMFpNNu9FsZGfiTpu2V3bZV5kGaOzdmF/r/UCDEulJtmNAR0y5pdbtm+IqEUq6Zj6gKCuaWfYE5zE0Cvf+ETg1qvjwKtiZQK04vaZCDAHELq4QQyAl3TOVwqL7YV2l0HRjiVSUspqwd4D8jX+St3adVQwBRP7+OwGvVVgFgt57j8jFvxE291ubc126dCFq2e8OxdCNxq0so27GV18R17sPOYuX2I0RTRaHYghAX1buQn8yi7z18TZiCECZ9AfLVgWwNe8ZEg0tuVBqIUC9knbBexj2XDvqtvajaY8Q6t1bj8ultu+np0rBQBc1qd8byVp4iuyyAq26FgG49mkOCgHXYb3QtW9E6NOeOHdpwh468KP/2xQ9dRKEMmuGfxMKDpykKq69e9u1VUvXZ+DF05IAUpT5/IS2xV6B1QKfuhDRzbbt9DppK+vcRijNl8TQs7Hw6mUIaVcx7lBZPbRmY2DY/5Ea+QzaLrZf7m3cnXASpKfS/u4qnB1YO5yEEfRwU9HCWUmUfxJqJyUD+odRPrIktiJNg2iyWMUQSJGCyctDADUqIQF31a+4q5YhCGVbny3GAqAOcLGzHt4K6rX1p/v9DajbtmL7rHHXIBqWbQWeP5zOL6/t44+vY1n8zl9cPiH9Du/8teIhbv3Xsbd20TL/amQL0b+cV3e/yrmcczirnbmvQUV02P6UigioSPdILuVfcni+h9aDzxrNwfzjFeZoXuL1tt8yqWQ0qR/+hefoerh2DLL5UPab1hJtuDuiRYRKn+8KJxXu7VXo3UvRl+2WlF7Io1acWlNz/+m1YNRD3J/SccPB0hfg6bWYD6+mtMX76O4ZZ1O4UTQaSf/4f2TPn28zlaBWo4msi7ZOFOb8Alw6dcS5nfRF59zJdmvO5/HHUHp61u4e/iZuvXuTKghgNGJKSSF15kxAxH3wYJQeHoCU36cq7v0jyN98GcOVAqlAapp9QdZ9hSZarN8FjLa2BWtO0dV9PnT7GJQKBk2VnI5FUcQ73I38PD3uVb6/BaDkREX9LufmvqhDXXHrHSllam4sZYMu9W3CloMfQXouc776AT/NFB4w/IZP45Hkv7keAKfmzVE4OaFt0ADvydce9n7DmLgKMs6AxQTzR0JROiT8BceXSf3Nx1T4Io35ET6r4uTceBiJV2D1AQFIxk8l4O2mppEo4onAQA81hWbRKoYKzSJxpWZaO9t+NIdrFDQyn4PW9Sn+q8LSWnzgDO6xg6HjVPT7j1E5RYIhoSJSUNciECH4dSmjeEoMuPiDbz3uBOq09OPUbimfVaPOQbh6aXH3dSI/U09BtvQDsO7LGPo/0oSCSpFyJqOF4nwDzjfLGfxvIIoiuWnFePg7o5C39v4RyILoX0xcThznciQfkc2XNlsF0a7EXTy1Vcp3M6HxBF5p/wotf6n4IH+t42tsvbKVYJdgXu3wKoaNKRQCGoOSnxp+TdbP0lN87srziHqT9MErQNBrHVG6SR9Mdnv7hiL4pAlOiHi6jSW34EGMSYWYcktROYhIs6LPg/NbpNdtJkpV1otskxdy5g9pHGCwRFE4bx+CUIqT4jj5pjcxHgrD8+QjuD7ytGQtADK/+cZODAEEfrCA5Lekcg4Bz01GHVhh/VFotYT83yckvfAiXg8+iHPHjnbn3yxUvr7oWrSgJCbG2pY6821SZ1aEl7sM/C8KnbSdpmvui3v/CNT+zhTFZmBOK7axHgDkmCzsLzJjFGHrldE2ffWG9oXQEGg80qZdEARGv9QGY0YJhiNpmHL06E/bh+0DqENdJRFaxdXlXJytFSvD4MQSl8eYrOmKKWM+Cg8PIhctRNDcAV9ySnVFBFrDwZJ1cvMbkHhIamteKQWFVyRM2QnflVWDb34/6LyI3lLxf5ZhEglr7Q9HK3xmXCuV9SjRJHOlwJ8sk5F+7rbbycXmAXC6klO3AkyFzhgNAurNb1JikKyjgkZANNg6wOt6doDgMsf3Jrb/p7ebsCbeNO8ViruvkzVzdd3W/hzbfMVu7OafTgGgUAoo1QqMejPxMRm1zsMkWkQSzmSzb/kFAqLc6f3gzUsWe3pfCtsXnKHNwAgimvkQVM/D5qFM5s5D3jL7F7MhfoP19aHUQ+SV5rH18larGAIYFDkIhaBg4ZCFtPRryX9a/4exDcfyw4AfmNV1FjqlzmZLLH/LZZtr5G24BIA2ysMqhuxIiYH3gymPmXY1LkYTJH0wZMyNofh4pmRRcsSZ9ZIfiG8DGP65lEUYoMszMD0RnDyhKAPT8X0UmXuRbviCYksfisyDyTK+jVGsD0Bu8SMkf5mAIamQ0kt5ZC9YjODkab2Mttl9uI36jsI9FVtJaZ8epeRUFhZDRbSL++DBND59isDXX7vlhVd1bdqgjuiKttVDoKjyLKPSIqjDAHC+vw77sg0kpUrWoKwq0WF5Avzl68yuQjM6L/tnIp2bmvpd60lOxQ7uUa1R4hziiueIuvg81ARXB19Gfk+0sPvwLywsJC8vjxMn7P3NMorMZP0pCV/3Af3vDDFUlcbDpX/LxVBAcylzc2WCW8HTRyRhNOILzGYLqRdtLaF1OwY6fM9cO7gR/pgkWooscLDQZDfGigBOrpJgKLF0QxQ1lJSF0PvyIkqPChXqHJSGOujmb+teLwqFQI+xDWjVL9za1qhzUI1WlUZdgmg3OBKA2O2JrP70GNsWnMagr+E9A45uuszaz2PISirk1J5kMq4UUJjjuITM32X7AskUfvTPy6z831HiDts7jsvcWcgWon8poihaBZFKUGESTTy3/TkOp9mGCrf0a2n9d+GQhTZ9lmIj6d/EYCkwWHPAGBMLHV5PV1MdpfX2vkk64wYMDMKcU0r2otN4DKtjrQNlw8kyx9Vm95ZlvB0Er8RLQkihgIZDsESvJtXwY/XXL78ffEj/Qspz4tL3I0SzEX30l6j82qEO7+7wnKxfTuHcNgDv+xo47DfnlZK99CwuHYJwvoZaUteD1+RHMOZI/jmCygn9kZ+szsjl9bxMhRn89p1kRbhyMouH3u3MXwlF9NAKOJV9wVwsMpGSkwcCDPD5imXZUwHwcDfwwDv9sVhEtLrafTQICgGPgZHEnc7m0qUCgtQK1J0D8XDXYikwUGIsRK1W4+zszA8//EBubq713FahoUQnVtQUW2Qyoho2FGdPT9wXLGDChAkobrHorJF6fW2PW9gnKAVstqKST2dTWmRC56Zm8NTm6IuM+Ee4Q4Q7bj1DSXn3oHWs5z2tbKZJNYkUtA2gnmopeSdD0RdK1j9nxTZcVWswFkeg53mKzd1QC5cRcUZJOhrhDN4lL5HBu3io5uE24a3r86G6jXgHu3Dfa+0pLTaya/E5NE4qG2HZfkgUJqOZ/SsvkJ1cRHZyEQBJZ3Nw9XKiz8RGePjZ+kWJosiBVRdt2pa+L4nbFr1D8fDXkXG5gIyEAka90Aan68yqL1pE9q2w9+Xb/OMpNFoVgkIqa6JU30G/2zIACOK1JDa5S8nPz8fDw4O8vDzc3d1v93KuitFi5ONDH/PrmV/RqXQ80PABfj75s924r/t+TfdQWyFgzisl789LqHx0GNOKrCUdnNsGoD+TjaVI+rJ17RFizTUEVB8xZjLA7Cgp/Bmg4xNw8FtMoh+ppfNshoZ+2B1DUiF5G+PxHFoHtXspfFxf8t946pAUal+VsxsoXDCPXNN/ruEdqkDlq8WUefUnxOBZXSpCmcswFxhInXMI0SA5GYd80O2mmsSLo9PJXlzhUOrS2R+lNh5jcgpFxzQoXMI5n5/LSYtja0AXFyVahcCuAhNmwN1Hy0PqIeSaAsk1heD3zAJcgmu39VCV0hITqz45SmZCISIiIGJRGMj2d1w3zNfHhz5ffY0AxLZozukmTezGeHh4MHXqVJydK77YRFG0vsdxcXHodDpCQ0Ptzr1p7P0MNr8JSg28cBpcas7svOPXs5zclUSTbsEOt2f0F3LJ+f0cnqProWsoBRzsX3mBo39epnX/cLrcK4kr0Wgh69eTqM99j4da2uq1iC4kly6kcp4nV+VyPNWV/q5C2sLj2/7mTd9+DCUmfv/wMK5eWkY828r6O7Dkvb/ITLB/SAtt5MXI51pbj4vySln24eFaW4O6P1CfFr3Drmutp/cls+2XMzWOaTMogs6jrp4+Q+bvcy3f37JE/RcyN2Yuv575FYBeob0YXne43ZjVo1bbiSGArEWnKT6aTv7myzb1rZxb+aFrKuXTETRK3PtFEPBcG1ApcG7tX334fPzOCjH08AboNxMAlZCBm3KxzVBTVgn5689QGpdL/sazkrO0xSRtTTgSQwB1elMs2hbL9L9fS9CjrhXHT7XCKSC/6pnSNSuLIQU4twsg8OV2eFb5sMpefBbRZBtdlbXglFUMAZRW2Rq5HvYsi2PLvFNYzPaRccUxtr5TRfvTce3TH8GlCwoXabshnuq3RmLUKo45a6zh2x3bSk7YnqpUIsc+dt1iCECrUzHqBSm8u8DjDJmBe6oVQwD1XVysPvdhVxxHG+bl5bF161brcVZWFh999BHLly9n6dKlLFq0iB9++IGMjAzy8v7+e18ruj4Lb+XC62lXFUMWi8jFY+kA1G3t2HroVNeToFc7WMUQQPthkQyd1oJOoyryWglqBb6TmuPx+APWNoVQhNbdNsLTeViVv/Umo2pxU3c+Gp2KCW93YuRzrW0eOkIaOM6dlHwuF0ulbfjdi89ZxZCzh4Yh01rUeL34mEyb86+F84fTra9DG3nh4Wdfr+7oxsvs/O06E9TK3DTkLbN/GaIoMjd2rvV4UNQg6nvWp4VfC2IzYtGpdOy4fwfOamdEi4ghsQBNqBuCQsBiMDusYeXSKQhtXU8UOhVFR9Jw6RSIQqNEEehC8IyO1SdYPDwP1j0nvW7/uLWkAV2fg72f4qFeiId6IRmGdym1tKJg5S7057UgKNGfzkSMf0my9Dcb7Xh+wFQkYDBLSfwCX+1g46Dt+3guotkEQj5uPV0RFv5KiWa8w3lUAc74TWlhFXauPjpcOwWTu+4ihXuS0J/KIm9DPJ7DJaFkzi+1e68yvz+O99iGOLeSwohLL+aSt/kyXqPqoQ64ug9HdnIRMVskcVCYq2fotJaotUpEk4WM749bQ+XVQS4YU6QtguQ391nPN3hoKM6VLHhOJRnodRVfwvf+ty2BUR6IokhBlp68jBJCj78odXZ5Blo5fl+uBa1ORXgrV46kZlx1bGil7N+elbbRqnLhQkU9vJ07d6LX6zl+/LjNmK++kjIxP/300/j63qRaXJURhFptQaVeyKWkwIjWWUVIo9onPVSplUS2qOY+IrvCAwthyYPQ6zV0Tt0prZRqQd2pDxQ+B3s/BWdfKRDhX0yrfmFcPJZBVCtfYrdVbL9aLCIpcbmENPTCbLJw6XhF9GPn0XWJauHLU9/2sclpVJnEMzksenM/E2Z1vqYIMWOpmeS4XAA8A5wZ8mQL1FolqfF5LP/oiM3YEzuTaDMwwupILnP7kQXRv4y43Djr6/pe9ekWIm3jLBy8kNzSXAQEnNXSFkTBrkTyN17CvX8E7n3DHUYLaaI88Bolme01oW6EzOoClT4gFE6VfoVilsDKKeAWBFN3V4ghgGb3VLzu/7b0s2IqxC5Gp9hDqaUVReedraH6oqAjqXQFgZpHUTWtdG4VykP+NVHudtFqKg8Tcd0qWcEEBQrPy4jFWWhDAnBt1YdSS1tUmkwCnhslPXlmXZCqnpcV1tQ187HWgSrcm4zHsDoIgmDjaK700mIue/rMXnwWw5UC3AdFkrfxEoYrBRTsSMT7AakukyiKxGxNwCvQBZPBTFA9T9ROShJOZnNgdcWXf9LZXPYuP0+v8Q3Rn82uyBukFAh4tg3ZS85SfKziSRTgbKqUWbtl5kI8Th9lf8e3QSEw6d2WaAOl0HxBEHD31eHuAawoS1NQCyuCyWTijz/+ICgoiA4dqk846d6oBCqiwtGW+ONUHIhRm4NgUSEqzITW80T13XtYgIhFCxE0Wvo+8QSp9esR9fTTFBcXU1xczIEDB8jNzWXp0qWcOnXqqmv87bffUCqVjB49mqCg2pW1uJmcPyr9bka18EWpvIHG+MbD4eULoPPGRRQwZRRTcjwTrzENpN/h8r+vuwBXLycmvi89aIU08GL7gjMoVQJFeQZW/d8xHnq3M5mJhZjLrLtNe4RQv31FgemIpj70ndQYi0Wkbms/CnNKWfyOZNnMz9STeiGP4PqeAGRcKeDk7iTCGntTkK2nZd8wBEFAFEXOH0mnIEuP1lmFyWjB3deJ8TM7Wq1ZgVEejH2jA5t/OklWUpH1+heOpts4k8vcXmRB9A/nSv4Vpu+ezqSmkxgQOcDqSN0nrA+f9fnMOk4QBGsBVpC+mPM3XgKk7LYlJzOt4satVxjGlEL0Z3Nw72O7jy5U98EuipIYAihIgd8r5Y5x8ZNqSVVlyBy4sh9dzn5yTU9XTGUxIygkq1OOZRp+3lHV3n9xtCQKyq0y5ZRER3Np7Lgqa7RgyZEKTAZNGgI+jTBs0eMmzkco6ChVQV/2iLQt0l/KUKwJd0cT6Y7hkiRITGnFqANdrELMc3gdXDoEkvTmPmvh0cJ9yRji0zGkSBEvJaezEU0WBJWCS7GZ7F1W8UQf0tALNx8nzuxLsbu3k7uS6DmuASXHK8SXW3fJX0bX1MdGEBWYRa6UJUxs12QT6vqFmNZ/AAiYN/aGye/bTn5+q1Sd3T20Vlmco6OjOXbsGMeOHaNVq1ZoqokCO3lKSsnQoUMHzm4tRF3shSAq0Rg9UaoUmE0WxAsWLMXFqEND0bVujaBQ0PazT1EFBKAJq/h9S0pKIiEhoVoxFBQUREpKxfuWlSVZAVatWsWTTz7p8JxbhWgRuXhM+h2pWrfrhlC2XScAnsPrWi2XdzN1WvlRp5WfjdVnwYz96MpyFLXqH07Xe+1zL1WuCad1VtNxRBQH10ifExdjMqyCaOX/jmIsNXOyLGeSm7cTddv4c3J3sk2ySJDSBlT1J/QJcWXsGx0pzjewYs4R8jJKuBSbKQuiOwjZh+gfRnxePI9teowfjv/Aq7tf5dntzxKbGcuLO19kQ/wGFp2W6isNjhpc4zzlWy7W4+Qiqw+MroUv3mMb4f9Ma5zq19LUn1il0OXlslpW3nUkp05H0UJO7vDwBpRCHoq8PdZmffQC6+tSfV2HBU0txUYSX9uDMbkIFAK6ZrZbDAnTnrI7p5wG96TiNGQqTv0GEVLvI1yU2+GTxrDmGWlA9K/WwpqCQsD/iZY4lZUHKDmRiSlbL22XCaBr4YegVhL0iA4lFQKlXAwBiHoTSTP2kvvjGtL2JuNVlncmUC2QcS7HTgxFNq+ofZa1cx0lxyWTi3Nrf9zLPjydGntDmFSE9azezLYCEybAVZGOk6IAZbfH8KnviVNpLpc/XEnh7or3l7gtsGSC9LrJyKtu/1gsFvburahN9sEHH3DhwgUSExPJzc2ltFSyjmVkZJCeno5CoaB3797cP6U/glixnTpwSjMEhUBugYJinT8eI4ZbUxc4t2tnI4YAmjVrZrcWNzc3/Pz8GD58OFOnTuX5559nypQpNmPS0tIwm29fYdD8rBK+nradotxSNDoVoY1vXY0wGclvpzIl+VIZnsa1LIbbZmAETXtI/nTx0RkY9Ca2LzqDsdT2dyo+RnpQ+WvtRbs56rSpPuLU2V3D8GdaAZB8Pg99kbHasTK3FtlC9A9j+u7pnMw6ycGUg3Z95aU3nJRO9AjtUeM8lbNLV0blp0Md5IIgCGh0rg7HOKQ8PL4q/d8BzxqegDxCMAvu5G1fgKBbB4KSoAktSN/wA7p2j4HKE/2ZdHLXXcacVYpLJ1fc+zWgJCYXypweVb46G6fu4qNHMWdXbP+F/zwPhZsbltxcNHl7UAaHSVXPQdouKs8rU+78XZQBVw5I/hpl6Jr6oj+dTcnJLISycFltnYrcS8r41fhoNpBjfAKj2NB6nkAJIpJTZWGcF6HkE+qm4mSJmaY6JWlGCweKKj5ou9xbj9b9w1n3VQyXj2eRvD4Gb0VPlK4iXvc3sD51CkoFp7Qq4nJtP0yHepWVIGkyAreRjSk68TEA6R++h2v3DZIlb9G9FSfUIknf3r17ycmpyIItiiILFlSI1sjISCZPnszJk5J1qG7duuh0OkIa6rhvejt+//AwXgHORDb3IbiuG0lx+Rzo+BYHT8PkGrIMN23alA0bKnJphYSEcO+99+LtXeGA7OHhgYeHB66urhQWVkQbXblyhaio6i2LNxqjwczpvck4uajZU8kCWLe1H6prLWIs87dQKhWMeqE1qz45Zm1zclHjHVy7XEwKpYKu99bjzP4U8jP1LJ99xBrWX5mzB1PJyyimpMBe0ARE1BzN5OGnwzvYhezkIn55fR/3T2+PZ8CtL58iY4tsIfoHUWou5WSWfa2nqkR5RFn9hBwhiqLVB8Z7XCPcB0RY+5zqe1176LjFAidXSq/HLYbIMr8drTvU61f9eWUU1pkOiIgl2SjUBjxe/JLAGQ9jTD4KQNb8c5izJCtE0YFCUmbMoajSdpF7pfDYooN/cXn8BOtx6Ndf49KpE7qmTXHp2hX1kP/aOhA3H+N4Uadty4U4NfYGhWRZK09GqWtR9hQoinBqNRpFHAHaF9EqKqxl3upPHE7fVCd9SQaoFaiAxuFXeOrbPrTuL4nH8qKWZrMUOqzzvGD9fxEtIpt/iCGuUjSLAiOP+E/EVy2tjfAuuI+uiEgyXL6CuaAAUqJtFxLa3vH9U35rIseOHatxzKVLl5g5cyY7duyQ7q1pRbJC/wh37n2lrTVUOlhTqfaWCMd3JFadzoqrqytdu3SjXr16TJ8+nccff9xGDFVm7NixBAYG4uUlWQdq43N0I9nzexy7l8Sx+adTVosE2G7HyNw6Qhp4MeTJ5tbjoU/VHFVWFbVWaS0aW1kM+Ue64x/hZj1OvShtpYc18aZp92BA8lOyy9TvgHJfJqPezJrPoq9pfTI3B1kQ/YPYk7THYbur2taS82qHV2ucx5j0/+yddXQUZxeHn/W4uysJBAghuLsVlwKlSEvho4W2tNSVulHaUqGFQvFCDYoVd9dAIJAAESDuLpvd/f6YZJNlE0hwmeecnJOZeeeddzebnTtXfrcATVYJEoUUk4Z2WHXzwizcGamZHIuaxBGvR1EWfB8m5A2prMG/GzSrMEiajADFjSso8o9UiA2amuL9++8AWHbpgq7I2BUNgEn3KoFIdQSl8QcojozkfLMwLo+vyl1ymDoVy243aAxq6SIkqFbiVKGHc3IZrBgJxwXNF5m5AoWr4fusD9MlnYScBFCYwbAFOCg+xk01EvdhFzCVHcRe8SFySe2NbHtbl9JO+rNBE1o/0xPI0WArF758zQr/ECwIIPlkFDHHqqpmnnUexrMuj2Ma1E5Q8p68C2RyZBYWBC9+CaWVWmj8uXVbleEKMH59zaHMCrRaLRs2bCArKwu5XM6bb77J8OG1GJDVCAoKMth28bXGwlb4HNhHbzM4dvlsJrWRGp/HxbUyvOStUKmu094F8PDwYMqUKTz22GMAHD16lOjou1PWrNXq9L24qtPjqUb6/BORu49vqCPPze3K1J+74eJnXe/zA1sY5n45eVsyZEYYI95siWcjQ8M8oLkT7UcE0uOpRnQYXrcecQ1aVSV352eVUFokhs7uNWLI7AFic5xQFeRp6UmhuhArpRUphSks6rOIILsgtDotxeXFmCuu7xqu9A6ZNLTTiw0aKDFfOSp0k+/5EZjb1zRFFZvehOx44feG/UGugtBRQhNJJ2OxvWvRFhZSsGcvAD4rlqPyqwpz2E/sT/7u65+fv+En8jfUfMzm8VqUhK/F3AEGfi+0CRnwLXwfLoTPYjYJP2UF0OY5TBvbo04UDDGFhwUylQ5yr1YZGQ16Q+NhSGQKJKa2gqds32xMc4+gcCokKuFj7OTGek1yiQUFeUMxuXJYyKsqK8Rk7RiCTb9DLvFHo8tDUXQANr8FFs5c/Ocq0A8Ad+VppJIKQ6rlM9Cgl8HckkaDsPL6kIwzCvLmf4hN94rmriMWgW/N6tyVHDlyhGPHBG9XYGAgKpWKxo0bk5ycTFRUlEEYTf9WmptjamqsuwKgLSpCvW8HXUo3YT5nORv+ziItIZ+8jGKsHIzPidqbiLZcx/mDybQfFlAnZV9fX19MTEwoKSnh999/55lnnrmjwo3ZKYWsmGkcvm7e25ug1i537LoideNWhFK9Qqq++wJaONH7maqcth4TGvH3l8fIyxAazfo2c0ChlNXrb25lb0rzPt6c2CS0Q/r15b1M+KI9cRHpFOaW0WqAr9j77C4jGkQPCEXqInZd3QXAFx2/oIljE7Q6LRqtBoVMuMlKJVIjY0idUkhJTDYW7d2RyIQS0cr8IdMm1RL/Lu2Ay4fBuy0sqcgrSYmECRuEm3RN6HRCs9VKKvNRJJI6VS4BFOzeja60FIW3F6pgQyVf67490eTvInddBOrLB9AVZWDevaqcuCRiGbURfPpU/fphNR9XpdkS2MswJ2rzW+DSBMuOHdCVatHkFGHdwxU2vAwnq3JpCBkivPbqeTmdX4fDP3O1yU/sPZuJqTSN0fbfosMCqaSYjLKPhNeibcvVueVAFvaKDzGVgadS8AilIMFbAhz6Ca1OysUSoU2JqTSbHtZVlYT4dTF+XZbOWIV5knEmh8KEYlL2FCE3t8EhsJfx2Gs4frxKNyW42t+mZ8+e9OzZkzNnzmBubk5iYiJ79wpG7ZgxY4zmASjPyuJSz17oSkpQeXjg3SMU9zMnSYzO4eKJNJpXC9sCaDRaLkUIn1N1qYarMdl4h9zAOAdkMhnu7u56/aLTp0/fMYMoN73IwBgKau0iqFFLub1l9tUoLi5m69atBAQEkJubS1hYGCYmoo7NncDEXIF3E3uuRGXRvLfh59PMSsmYD9tyaPUlrBxNMbW4ud57bQf7I5NJOLohHoCVHx7RJ1l7BNtiaWdS48OCyJ1BNIjuc7Q6LZEZkSTmJ1JcXoy7hTuNHYQnFalEivQGX7xZq6JRJxciUckwC3VEnVaEJqcUiVKKaWU1xuVDsLQG8cOU0/DPJHhilfGxwkz4qkpJl7An65QvdC15m7cAYNWrd41PQ3aPd0FuWQySYHRqNQoXE5LffR91Uga6MmMRSaWvLy7vv39rzUEb9DFOEj+3DolvJ6x7ecIvnWBeAlS7frHUGRP/Hhi9guZjoflYYhYIuV+BJvswlZ2Gjq+g6/ASFptSKDhoWGWWqX4Pez5EKRFyoy4UmBCitMdClkliWQjFWhtMJHmMf7oImc9WWDYUWk4Eec2vWdX5cZRrfqQsX0FqnC0XAwPw2H2ABi1aUFJSUqNmT0FBARkZgifR3d2dRjW01qisAvP19aVDhw41XltbXExpTAxZS5ehLRRyMSx79UIikRDQ3InE6BwuHa8yiDKuFmBhqyI1Po/SwqpKvbhTGXUyiAD69OmjF2uMioqiT58+d6Qn2vofThtsN+7sfsf7U23bto0TJ05w4oSQX5eSksKQIbULl4rcGr2faUxpkVof8q2OVCrRt1a5FfzDnfQGUfWKs8qk8P7TQvFuXLfPvsitIRpE9zkLzyzkuxNVXoDePjUbDjWhTivSl9fnrL5Izuqq6heTBlaCwnRKJCzsXfskMZugIB0srikjjVhe9bt7OAz6sU5rqo62qIiCPXsAsOxT+xqs+hpKCPiuXUPckKGUXcpHFRREaUWuiPv3c7Dq2bPe6zAiZDDE7hSSwo9UqH5HrYU+X0DCAUg17NZ+qaQNm3Jep83OdML7GHrodDodUfuSuHBU6HTdoLU7eH4MracgkSmwGRSAJreY4qgcg/My1e8BUCKVkKuBI8VjkWmLKNYIuRB+IabIWgwWBk83vDEb0XYalk3Xkrk/hSOtW5Ho4UHk/v38t38/UqmUqVOnYm8vfOEmJyezdOlSioqE0JqrqyuTJk2qx5tXRXlmJkmvv0HhPsPcN8seguHsF+bEnpUx+rBZ+uV8Ns0zfG9tXczITiki/lQ6ulEN6pSs6ujoyDvvvMOsWbMoKCjgr7/+YujQoZSXl6PRaDA3v/nO7/v/ukBsRDp9pzQhJ7VIv3/g9GY3ladSH7Rarb6Sr5JTp04xePBgMbRyh1CoZChUd7ZK0N7Ngl7PhLDl15oLZtb/cEpffSpyZxENovuYd/a9w7+X/jXYV5u+kE6nI3vpMgr370fp44PS1xepdXitc5tlzQfmwKmVNQ9wCxOShQG+aSS02+j4Migq3LfVQ2Xtp9ftBV1DwZ696IqLUXh4YFKDB6I2pEol/hvW39Q164RcBUN+Fn7v9TF8FQAFKXDlEJz522j4mSLBmDuzO5Hmvbz1N+3SIjVbF0aRcEZIHLZ2MsXp8RlGuj+qBvZGBlElGk9LyMrkXEFng/2BPa5fHWaATIHF1NmkHhpL4jXhI61Wy/fffw9AQEAAV69epaSkRH+8esVYfdAWF3Oxazd0ZWUG+x2eexbTsGaAEHZwC7QhMSaHSyfSa6w46/B4IJt+OUNhbhmp8XkGRodOq+PElgQcPS0N8j0A5HI5DRs25OTJk0RFRRF/pIBSVQZycw3Pv/A8Fhb1kJSooLS4nIiK1iqrPj6q39/5iSA8g2uufrudXPu3qSQ5ORk3N7c7fn2RO0dgC2fiT2cQcyS1xuMH/r5Isx6enN5xlbP7kug/takYSrsDiAbRfUpKYYqRMQQQZCtU8OTv3EnxqVNITc2QmplRcu4cuf9UhHl270bm1BizdrV/SZpkroBfTgnVYZVI5aC0gOcOCdVX+76B7R+Apgz2fCkkF/f5DPKSBVVnEDxDjQbW67UVHjlCxk9zKTp0CADL3r3u3ydcuRKCH4NTK+D0HxBl+Dcp7reQxN9sACjILiXuVAYypRTvEHu2/lZlDAF4NbSr8XWah7tQdrUAlZc5chs56Qtj9MccOrrDKcNKLFMrJW61NLWsDdNmoWS0vL4RdfHiRaN9NYXKbkTppUvEPtbfaL9Zy5Y4vvCCwb6AcCcSY3KI2p9EYa5xJ3KPYDt8mzkQcziVmKOpBgZRzNFUDq0RKhHHf9YeC9uqSjSdToeisMqrWWRxGRA+yrNmzcLT3ZtSdTFPPvlkrR2wdTodcacycPC0IDe9mIIs4/W1HuhH4043rsy8evUqtra2t+SdqvQOOTsL1UlpaWnodDr+/PNPxo4dW6skgciDQdcng2kz2B9LOxPWfX/KqAIzLT6ffX8KrZkOrblEr2eMhUtFbg3RILpP2Zqw1Wjf2IZjKb2US1n8FRJfeEHoBF8DEqUFZu2qbjxmzZ0oOiFo1tjIf0AlPYdEoq7SpFGYwasVN0NNGZhW3GxDhggGUSWHfoLen1Zo9OgEDZuwJ+v1usouX+byuPEG+6z69KnXHNVJTU1l2bJltGvXDm9vbyIiIvD19SU3N5fWrVvfHkMrZIhgEB3/Tdg2sxckBZTmxJa2QaerKu/+7xeh8Wj38Q1JiDT8Qgvv61Pj9BKFFLvhVVV+rm/ZkPLNCRTOZtjVkDfj6m9dr4aTABKplPQ+veH8eUKSksnWaUlyv/6NvGnTpnW+yZZnZZG1dCl2Y8aQ+ulnNY6xGmBsJFWGzSrDT3Zu5khlEjKuFODf3BGpVEJgC2diDqdy8XgaHYYH6PPmKltjACx+cz9jPmyDjZOgv3V2bxKxe4pRWTtRappmdN0riUJlz549e+jf33hdABeOprJ14fX1jPyvo0hcSVxcHIsXL8bLy4unn35avz8lJYXs7GwaNmx4wznOnj3L4cNCAne3bt0ICgri7Nmz/Pnnn2RnZ/Pbb78xY8aMG84jcv8iV8qwtBPCcz2fbsTF42n4hjrwz1fHycso4a8vqvTNEi/koNXqKvoM36cPkw8gokF0n7IlXkg2bunSknfbvMvh5MP0Le5Exq/CDVcVMozSyBqSnQG5W1WFl0Spw2ZQAFJFMWYnJ6CUxhmf0KAP5YVllMZcQOHujtKjwiCy84V2z8OB76vGpp+Hs2uE30Pql8ypTknhUi/jXCGTGlo01IWMjAzmzp0LwObNm/X7jxwRmjNaW1vX6WZzQ/y6gIkNlOQI240GQ98vALj0nRBWdPW3JvlSrv6U7YvP6X/3CrGn/7Smdf7iklmpcH29JRKZFIlEQocRgez78wJypWAItOpfdwXmtLQ0VCoVpqamxFR4gFo/9yxF01+i+MhR1N26Etu6NefPn9ef06ZNG/rUwUjVlpUhrUheT3rjDQr37CVr0WJ0xcVVr33xYuQO9hQdP45NDRpG1cNmIPT9atjOlTO7EwnrJeRMeDayQ2UupzivjMSYHNwa2LBi5mHy0osN5oral0S7oQHsXhHNmT2JSJBglRtMjqwUtTL32ksDEBsbi06n0/9tCnNLObohntBuHkTuSqzxnOZ9vLl4PI3w3t7YutzY43PwoOBNvXz5MnPnzmXSpElCw+VlyygoKOCpp57C29u71vOzs7P5888/9dv+/kLfsgYNqozo/Px8fvvtN8aNG4dMJipjP+iYmCv0nsfmvb3ZtdxQU6sot4y5z+3E3t2CEW+2QCavezJ/XkYx6jIN1o6moor6NYgG0X1ISmEKEekRSJDwecfPcTJzwv6olLxtVQKCSv/ulJ75C3RVbR9UjRri++efZPwWRenFHGFn6X6kqk7YOO2BSmPo6S1wZB6c+UvYDhnC1eemUlyhSBywfRuKSu9Br4+FnxUjhQTrja/C5QPCsTq0fahO3nrjvB+X99+76SecRYsWXff4+fPnb49BJFcKpfiRfwjbjYXWF8UFZVyNzgGg85ggVn54xOjUlv1962XAVFLZaBegaTcP/MIcsbBVodNRZ+9QdnY28+bNw9zcnB49elBeXo6trS2eoaGUzv2JuCFDMd2+gxEffYRs1CiWLl3KlStXrtvNXj/377+T8sGHePzwPRadO1NYoSVV3RjynD8P89bCXCr/2puP+oY66g2iBi2dsbQzoe2QqvEymRT/5k5E7U26rqLvyS2XObnlssE+U0sFzfu3ZfOWTZiozFAm+5FnV5W4nZWVxalTp2gYFEJafD5bFpylpEBNQmQGJUU1e2DbDvan7eCq9el0OhISEnB1dUWhUFBQUIBWq6WoqIjS0lIuX65aU2pqKjExMfpxAJGRkUYGkVarpaSkBDMzM4NEag8PD+Ry4bOhUCjo0aMH27YJYpcJCQnExsYSGBhY63sk8uBRqZhdiY2zmd6jmplYQGJ0tj6HrrxMw+Zfz2Lvbk6bQcb/cyWFapa+Ixjo9u4WjHynpehhqoZoEN2HVHqHwpzCcDJzQleuJW/bZaNx3iu3YBZalSeUm5vLiSMncLxUjBQJBVveQiItQVs2A2nUGmFQv1ng1Ro8W4GmFG12OrEvfI/6alVCa+6GjThMvqayqOFAwSCKF258eLQE6/rpu+T9t6lq7SuWY9a8blpFNZGZmWnQu6omTp06hbW1Nd26dbvp6+gJHSkYRFIFeLUFhFJwnVaHg6cF9m4WNO3qwemdhonB1dVobxaJRIKlnUnF73U/LzIykvLycnJzc/n7byEZPCQkBIlEgio4GKWvL2VxcaR+8imuH3/E6NGjUavVtQorVqLT6cisMEYzfppL8nvvG40xbRGORcfrCz9WEtTGhXMHknD2ta61n1ODFs41qkG3HepP447uzH9pj9ExmVzK0191pKysjPSMNHy9/NnzcyqWOcEUmyUhlUGZLI81a9Zw0uoSRTFVeVkF2cb5QgBdxwYb7Tt9+jSrV68mNDQUc3NzDhw4cN3Xe/78eYOGxceOHSM7O5snn3xSf3PatWsXe/bs4cknnzRoQzJ48GCDuTp06IClpSWrVwvioNHR0aJB9JBh5WBKk64eRO68SqOObvg3c2Td96f0x3cuP8+g6WHYOJkRdzqD+NMZxEdm0KSzB+Y2VXl1qfF5/PV5VdgtM7GAzMQCHDwsEREQDaL7kM0JQvint48QXiqp9PYAmtyraLJiUfp2ouhEFiZBTkh1BRQfmMdvp2Tk5OXTUupPuI0XMhM15VkFFP78MpZZxwGJYNiAcGcduYy8v/9GffUdALQSCVqplNTvvkNub4952zZcSVNw/mAyclkjGpU2wUMlhOwK1E0o/v4HVMFByG1tkZiaYhIcjKQWd31ZfDwlZ8+CTEbgnt3I7W9NV6P6U3OXLl04ceIExcXFODk5kZKSou92vmfPHpRKJa6urvpQw00R0AMeXwL2gfp2F5eOC7kp/mGCxH+7YQE07uxOzJFUjm2Mx8nHSp/Tci84d+6c0b7KJGmJRIJV375k/PQTuatXo/TxweF/k/Xeh+poy8q4Mvl/yO1scfv6a0rOnEGdIBjoJbX0DHN+7bU6r9PEXMGod1tfd4xrDS0wWg3wJayHFxKpxOCpuZIRb7YAQKlUMnCg8Lm3f8UddLBpXiQ5hSmU2Qqfo4S8SEwt3bHIN/yMWIbkUSrJZeiwIeg0UuzdjUNklQ1oT506ZXSskrCwMIqKioiOjiYyMtLoqfzSpUskJibi4eGBVqvVi2Lu3LmTpKQkJBIJM2bMqLE6rmnTpqSmpnLgwAGOHTtGYGAgVlZWrFq1ii5dutCsWbNa1yXyYNBpZAM6jRRCpFqNFkt7E/IzhYrDgqxS/vj0KJO+6UTcKUE7DB1cPJ5GaPeqPo+7lp83mvfIujh6Ph1yx6UFHhREg+g+I7kgmdPpp5Egoae3oKlTHCl8yDW5Vyk++B1SSzeUvp0oic4mfcEZbOy/48tzXugqvmSPKi6hyz9JC/tEsrMsyN+4Dss2gHd7sDT0WORv2w6ADtg7aiQpOh3K0lIGvvsuUldvdga+oh97gQ+Z7DQKbUE5V1Zu4locpk3DcdpU/XZlboZOqyV37ToAzNu2vWVj6OjRo+zYsQOAgQMH0rx5c7p06QIIoQaNRsOaNWv0RlNlSOHJJ58kIOAWhNSqhQhLCtVcPS+0rggIFwwimVyKrYs5YT29UJdpCG5z71o3ZGVlkZycbLS/ugijZa+eZPz0EwDp33yD/eRJFB05qu8HZzVgAOXJyagaNNBXBJq2aIE6IaHGa3r89BNmrVoikUqRmt1eQ1AqldBrYghbKgQu2w7xJ6yXl96w6Pu/Jqz/4RT5WSW4BljT939NMLU0Fqp0C7ABwNnXmqLTpUg0CnQyQQyv2DwR00IPrKwtSVWfR6KTk54ZD8ClK+fx9/dHIjE0SCrDYjeiSZMm+Pj4MHv2bAoKCtDpdNja2uLh4UFkpPCQcfbsWdzd3Tl27BiFFSKWiYlCHpOPj0+tUgESiYRu3brpPVO/V/QDBFizZg2hoaFiWOQhQiqTMuzVcEqLyvn9QyHRXl2iISkmx6CqNeZIit4g0mp1Bg8MDdu5cu5AMnGnMtix9JxBW5JHGdEgus/YkrCFjme0PH7cBKtOBeg87Sk+KxhEpad/R1eSS7ZHYypvN+or+ZxLTUUnNcxBOCa1oqOnmuwLkJ9oglYtAc/O+m6+e/bs4WpCAk0PHkSjULB9+DDyKtz4ZSoVKa4uSDSC8aRDR4lZElKNCbF9DpG6/0+slCsoMjMjxcWFoOhopDodGT/8gMNzz1Jy5gzxj1d1Wld4e+k9ClYVzTdvltzcXDZsqGpedm2OkFQqRSqV0r9/f65evUpublUy7bFjx27NIKpGbEQ6Wq0Oe3cLozCP0lROh+H3NmxRGWbx8fHB09OTqKgo+vfvb3BjVAUFYdmnD/mbBOO29MIFrj7/vP543jrBiC06VuVmT/3wo6rzAwMpvSCUAdtP+R8WXbvc0RtvYEtnfYfwa7FzM2fcp+3qPFfLx3yIP52BXUYL8q2jKTPJAsCrVyloy4k/bRj63LhxIwBDhgwhNDRUvz86Otog/AVCc9vqjWXt7e3x9vZGKpXSsGFDjh4VNIyaNGlC+/btKSwsJDY2lrNnz+Lo6Ki/VnVuJH8gl8vx9/fXtyypTnp6Ok5OTjWcJfKgYm6jwtxGRYvHfDhWoXK9c/l5yorLUZnLKSvWkJaQT05qETbOZqTE5lJeJvQ87PdcU9wCbTh3QHhgungsjc6j1ZiYG/dZfNQQDaL7gNg9e9jzx3/I5Q1Ikv1J6yhnpPk5pH78CfbPvYeuRIO2JBdN5kUsunVjh24YhSUaHE1LSJFmc1X3GJBuNG+igxtykwI0JTKi/3ZF+t8qklx2oysqYk8XQegvvVNHXNVqvTFUyRVPT0wLhRwftTKHAivhi/avdYmoVcU4dmhPerUv2YYVVUqXx41HnWZY5lxpDAFY9uher/fm6tWrODk5oayoZqqeT2FhYVFrvoupqSlTpkxh9uzZqNWCByA2NpY//vgDS0tL+vatWeCyrlyqkDEICL9x2XV9yM/P5+DBgwQEBJCSkkLr1q2Jjo4mKiqK5s2bEx8fT0BAAF5e11etrfSONW7cmBYtWtC9u/H7LpFI8Pj2G648V0bBjh3krPoDbV5endYptbbG48cfiH98JOadOuI0fXq9X+u9xMnbihFvtuDPz45hlduQYtUFCiVpREREXPe81atXk5ycrK/Cq3yf/fz8iI0VNJE6duxI165dyc3NxdvbG4lEoq/8CggI0BtETZs2RaVSMXr0aL766ivy8vJYu3at/lq2trZkZ2cjkUjqVBwwaNAgZs+ebbQ/KipKNIgeUlr198XR05L/fo4kN00oaPBv5khBTimXz2Zx4VgqLR/zJb4ilNaglTO+TR0AoVjj9A7B8N/3xwXUZRo6Ph5YY5uSR4U723jnBuzZs4cBAwbg5uaGRCJhzZo1Bsd1Oh3vvfcerq6umJqa0qNHDy5UPJFWkpWVxZgxY7CyssLGxoaJEycaJduePn2ajh07YmJigqenJ19++eWdfmn1YvWatcQ7qEimHOVVKRlWZhz1cyPv1Cmylu8EoDzpBDYjH8dm5megg3MlWv5VHGO/IpoEmWAM9VRb8bJ2AwE6wQCJkjbAyrNK2TbZ0oLdrVrqjSGAbDs7opyrnrrdK6rLEn39ybAXvoS9OlR9TNSqHAADYyi1Zw+sKrRcio4dQ11RVaMDNNV6SFkNHIDMsu4JfJGRkfz6669s2LCB3bt3M3PmTIPy+toaiVZiamrK22+/zXvvvYe5uTllZWVERUVx+PBh0tONDci6UlKo5uo5IVzm3/z23mj++ecfDhw4wJIlS9iyZQsbN27kjz/+4MyZMyxZsoQ9e/awcOFCI69EdSrDZXW9kVpVtE3JXrHiuuPk1T4nFh07ovTyIvDgAdy++KKOr+7+wtHLkkYd3Aho5sKUGWPrfN6hQ4coKiqiuLhYbwT17duX0NBQmjdvjru7Oy4uLgQFBWFiYoJKVZXY6u/vT2BgIC1atMDBQbgxKRQKgoKCDK6hVCr1+kgNGjSok7K2lZUVM2bM0OeBVapXX5tLFh8fz8yZM5k5cyaffvrpdXOfRO5vJBIJno3skMqqPLO+oY56T+rJLZfJzyoh7rRgEPlUGEMAHR9vQJvBQj/K6MMpxJ5MZ/GbB8iqaPf0KHJPDaLCwkJCQ0P1jRiv5csvv2TOnDn8/PPPHD58GHNzc3r37m0gXz9mzBjOnj3L1q1bWb9+PXv27GHy5Mn643l5efTq1Qtvb2+OHz/OV199xcyZM5k3b94df3114fyOQ+RbCV92pSaGOR8pChkgfLDLE49h9+QY4s4KIaByWRHl0qqyYCutKXm6cMxbdaOrROgdFa1ojMlEoSfWZS9PdnXtet21tGrViqeffhpzMwvKdRrKVFmYFiZx6ZJxMl51UtPTkVbzQGikCnRISH7+ef4ePoxkFxfM27XD/u23mTdvHitXrkSn05GSksKiRYu4cuVKjfNWflFHRUWxc+dOg2Mvv/xyjU1Ja6IyVFGdM2fO1DLaEK1Wx5rZJ/hn1nG0GsHlXBUuM6+TDk1dyM/Pp6ioiLg4Q52o6h3nq5OSklLrXNXDZXVRRrbo2hWJQgEVRpbj9BfxXrEc2ydG4zl/PnJHR1w//4yAXTtRVpSH244SQqISieSBzU+RSCR0fTKYPv9rgqWVJS1atNAfUyqVKBQKAgMDa6xS/OWXXzhy5AharRYnJyccHR0ZMmQIAwcOvO77IZfLGTNmjJEYZPUWKZ6enrz++uv4+/szbdo0hg4dWufXZGlpyTvvvMPMmTP1VWupqanMnj1br7lUXa6irKyM1atXX9fAFrm/UShlBmF7j2Bb/EIFz7W6VMOStw6Qk1qEVCYxapAc2MI4BP3np0cf2c/DPQ2Z9e3bt9bQhU6n49tvv+Wdd95h0CAhmXXJkiU4OzuzZs0aRo0axblz59i0aRNHjx7Vf5l9//339OvXj1mzZuHm5sby5cspKytj4cKFKJVKQkJCiIiIYPbs2QaG071ix+btUBH1KVdp0MoVSMuFEE9eQCckSnNKtDq2NpmO8sckVGYKyhS55NobPtU5lNsTl6XhiufLeNu6Y3OolJz8QraX25DTri1XagixOKalGXh6+vXrB4CNwo1CYigxTSHNLYuSG+eMEq0wI6BvHxL3nuV48xl4y69wNP0qSKXs6dIZk67dWPjNNwAkJSURFRWlF5tbsGABTz/9NGlpafj5+WFnZ0dRUZH+6bsy5FWJq6trre0WaiMkJIRj1XJhTp06Rdu2bVEoFNcVskuJzdVr5Jzcepnow6lkVzxBBYTfekk9CDkev/zyC+XlNeve1MQvv/yCnZ0dTz/9NBYWFnrtGysrK30Yp659yGSWlph36EBBhdFp2bs3Kl9fvSxC4N6qknavhQtQJyVhVs14eFjo378/ffr0Edp+KBSUlpYik8mQSqV4enpia2vLqlWrSE5OJjc3V2+k32y/t+r4+/ujVCopKyujSZMm+s9kpRfpZjAzM8Pb25v4+Hjy8vJYsmQJEyZMqHFsSkpKnR8wRO4/Oo9uwNo5p2gzyA+5UvjsWNipDNrNeATbojQ1vOXX1A+tXK0lNT4PW2czVGaPVl7RPfUQXY+4uDhSUlLoUdEZGwTl4datW+uVXw8ePIiNjY3Bk12PHj2QSqV6mfuDBw/SqVMnfQ4KQO/evYmOjiY7O7vGa5eWlpKXl2fwcyeIORdBvly42Ut0gERCiXcjZFaDKbNxIM3djlRJDjukpymwjKW0tIz8rBJybSMNJ9JJyc51QQdcOpmJpP0LhDRtBkDkmTM1GkPujo48lplFWIVbfcCAAQCc2JxAYazwtFFmkkVJxf+L36VLDP/jTzruOo9DSgeci8IJc+6DeZ5QphxxIhKPb74hvdUodFIFl5SGX+Rbdu4w2K6uvAuwcOFC1q9fz9KlS9HpdERGRqLVag3G2NnZ0atXL0aPHn3jN/cavL298fDwwNbWFolEQk5ODp9//jkfffQRxdXEBK8l9kRVaO3Qmli9MQRV1WX1JScnh2+//ZZt27Zx9epVfvzxx+saQxKJhIkTJ6JQGH45ZWVlsWHDBtRqNfPmzWP27NnMnDmzXuGySiqT3U0aNULlW7uQpMLdHbMb9ER7kJHL5fr3WaVSIZfLkUql+Pr6YmNjw6hRo4zOuR0GkUKhoHfv3jRs2JCmTZve8nyVXKs4vmlTVXVo82o6YFG1yCeIPBi4Bdoy5fsuNOtR9V3ffbxhIn5t4f1+zzbBxEKBa0BVn8C/vzjOry/v5cKxmpvNPqzct0nVlSEBZ2fDp3BnZ2f9sZSUFKNkQblcjp2dncEY32u+4CvnTElJwdbWuEnmZ599xgcffGC0/3YTs+UQxQqQ6iSEanw4KY9Do1SQ434V8CGFEtZRFTLRSTSYF3iDtMpQcE1vRKnOEplWyFOIO52BplxLSEgI+/fvN7ienZ0dEydO1Cvgmk6dikd5OeEpKbi7u5OTWsTB1ZeQY4lUo0Qrq+pU7nHlKjKtFm9nCbGmSsrypFw9VYRK6kCh5SVyitI5eySeBK0PAMWKm8vRyc7OZu3atZysUM2uXjkzePDgGyYT14ZUKuWZZ54BBOOrunrwF198wZQpU3BxMSyT12l1XDxh3AerktpEBG/E3r17ycnJYd++fezbt8/o+MSJE1EqlaSlpfH3338TFBSEp6cn06dPR6fTMWvWLP3Yc+fO8cknnxjN4ejoWK9GolaP9QNNOaaiZs11sba2pmPHjuzdu1e/71a8ONUJDw8nPDz8tsxViYuLC8OHD+evvwRV+srvxUmTJuHu7o6vry9///03Z8+eJSwsDDMzM0xMHt2k2ocJjyBbhr/RQi/G6Nes5gIQ31BHJlaE2C6dSGPTvKp0gi0LztYYVntYuW89RPeSN998k9zcXP1PbTkut0rzHt1oVuJGE403QeW1d6avpMQsmVJVhn57HH/RoIkzMq0KKwcTzKyUlBaVc+VcVo3u76ZNm2Jubo69vb2+Oksul+Ph4YFEItHf/CVIsKzm4fGKT8A5NRWFlxc+P/+EX2jVMZlWhUIthK82rhRCKzp0lJoIBlHb1u0N1nBtW4ianoYrjSEQvHkhISE0ahTClSOlxEbcfDJ0JZ06dTLaV3nDqE7ypVwKc0qRXyNaJlNIGTS92U1du6SkxKgw4Fo8PDxwdnamcePGTJgwQR8yNjc3x8LCgvHjxxv0saqJ+orxSSQSrAcN0ucIidRO9+7d9Z7rW61WvBs0btxY7wEGwUiqTLhu0KABMpmMrKws5syZw6pVNfdHFHkwcfaxotOoBvSe1LhOZfXeja/RiNNBflZJzYMfQu5bg6jyaT011dBll5qaqj/m4uJC2jUl3uXl5WRlZRmMqWmO6te4FpVKhZWVlcHPncAtpAES93JalvtjgQmNy2/s/Si1ELwlPdmDnyyV0EEtsHMzp1V/X71LdMOPp0k4k8mgQYOwtLTEy8sLR0dHg9BiTVyq5g0ZPLIvSqWSliEhdMnNxaJ1a3xWrURua4t/tVCRd2N7nK2Fm2ipaSr5lhcpdYhHKytDopUR95+U/03+H5aWlvTu3Zt+/frpK2omTJhAv94DmDJlSo15PB4eHkhKTBn42GCs8xoSse0K//0ciUajNRpbV3JSi4jdVULHJv0NwrEZGRlGn5OLFe5i/zBHOo9ugIWtisffbsmU77vgEVy3LvDVKSkp4fPPP68xBDt27Fjatm3LU089pU/KlUgk+Pj4GEkL+Pr68sQTT9TqTejcuTNt2rSp9/pE6k67du2YOnVqnfq+3Q9UN6BbtGih/4ypVCoDba64uDhmzpzJzz//XK+cthuh0+ke2UTde02TLh51Du/LlTJGvduKvlOa6FXZLx6v3Uv+sHHfhsx8fX1xcXFh+/bt+qfdvLw8Dh8+zLPPPgtA27ZtycnJ4fjx4/qbw44dO9BqtbRu3Vo/5u2330atVutzA7Zu3UpQUFCN4bK7TfiQfhT9FEOcKokS02M0z23NCUUcVlpTWpb7s115BtvcK+RLLCi3skUtE75UQoiBUSuwczVn9HvCa7W0NyFyl6ArseHH0wx9NZyhPTywtDe5Yb+anLQiMq4UIJFKeOrL9phaKHnrrbeEgyNGGIz1bGSHb6gDJhYKuj4ZzKm9llzdHkm5opByRSGVzxPKUns0ZRKK0+TMmDFDf/7QoUPJy8sjK1bDgs/30mlUA1566SVUKhULFy4kOTkZHx8fWgf34I9PjxqtNSk6B89G9TdIANb/cIrcii7pLfr5MHTIUP5Z/Q8Ac+fO5Zlnnqlon6Dj4knBGxXYwhnvxvY07ly/3m3XUr2jvFQqxczMjIKCAsaNG4efn1+9W4v07duXoKAgLl++TIMGDcjJycHExOSG3iORW0cqleLoeHs1qO4klpaWtGzZkvT0dJo0aWJwLDg42EBIEoTQ2oULF25Lc+Ts7Gx++eUXgoODjXqxidx/2LtbYO9uQWFOKXtWxnB2TyIF2SU0au+GvfuN5R8eZO6pQVRQUMDFixf123FxcURERGBnZ4eXlxfTp0/n448/JjAwEF9fX959913c3Nz0/1QNGzakT58+TJo0iZ9//hm1Ws20adMYNWqU3iX8xBNP8MEHHzBx4kRef/11zpw5w3fffcc3FRVP9xoPDx8yXzLBbOkr2OcWkK4ZhZfWAUudKSrUjCr9FxN5FiuzO1FuJRhw7g6W2Ey7ajSXi7+NwfY/X1XlHz37YxekstodgpXeIY8gG0wtjFseVEcmk9Lv2apQV6OWXvy3xZZSuWGSuqpEuGFcPJZqEL9WqVTkJ8K234REzj0rY2jcuSsSiYSxY8cSERFB8+bNWfGusTEEcCkivU4GUVpCHie3XKbdsAAs7UwoKVDrjSGAYxvjCe3uSXh4uL68fe/evYwePZqkCzkU55WhMpfj0fD2GM7V+6898cQTt6yaLZfLadCggd4Autn8KpFHg8dqUYlv2LAhhw8fNpJyiIyMvGmDSK1Wc+jQIcrKyvT5VhEREfTo0aNOmkoi9x6/MEf2rIwhN72Y0zuucnrHVQa/FIZ70L13JNwp7mnI7NixY4SFhREWFgYI2jJhYWG8956gnfPaa6/x/PPPM3nyZFq2bElBQQGbNm0ySPpbvnw5wcHBdO/enX79+tGhQwcDjSFra2u2bNlCXFwc4eHhzJgxg/fee+++KLmvxN7RBY+GfTGVZOKo+BAHnRUqFJhIzxKsisXLLAcLdT5UVF2FhNXcCFMqlTDwhWY1Hps7ddd1Y8GXKqqpbkZoUGkqp0MP48qjx57sAMCFY2lE7UsiM7GA0uJy0hLy+O9nw0q5zERBTNPMzIx27dqhU0spKVAbzQmCDlBKbC5HN8SRfjm/1nX9+20EF4+nseStA/zz1XEWvLLXaMyp7Vdo2aIq7JGQkIBarWbL5q2UmKTh0diSzZs3kZmZaXTu9SgrKzPQy8rPz9cnh0+aNOm2tRAREblVTExMmDJlCm+++SbNmzfXF6rExMQYfIbrilarZfv27Wzfvt0g+RyEh4KEhAT++OOPO1a9K3J7MLdW4ehlGFlY883JWr+XHwYkOjGwe0Py8vKwtrYmNzf3juUTkXAQfhNKZIs07SkoH4ydYhZyaSq0msy2Kx4cORWJxsIa06RYRr/7Ke5BNT+9pcTm8veXNQv6Pf5WS6MPeW56McvePSiEy75oX2NTzBtRVFTE3Llz0el0yOVywsPD6dChA0vfOajvynw9AsKdCGrtgp2bOecOJnPpeBrZKVXNCL1C7Oj3bFN+e20fpUWGuQ1TfuyC7BrvV1FeGb+9ZlzBBUKX9IQzmaTGCV/ITTq7Y+OhYOP+PygpKcHbuQEJqTGgk+LvE8ilhGgCAwNvqIxdiVarZd68eeTl5TF16lRycnKYP38+AE5OTjz33HN1mkdE5F6g0+n44YcfyMzMZPDgwfVK0I+Li2P79u1cvWrswQZBPTspKQkQ8pqeeOKJ27FkkTvEnpUx+jSMSjo/EUTjTu73aEX1pz737/s2h+iRw7M12HhBzmXMZPsxk+0nqHwFx97qhaWJggZnTnNqx1bIFNzaf3zwJi+tWFPjVC5+1kz4oj3L3zuEulRjcGztdxFM/Lqjwb7KcJl7A5ubMoZA8OxMnz4diUSCtFq7DvcGNpw/WLuqctOuHpzeeZWLx9NqTN5r+ZgPSlM5QW1ckMml+DR1IPqQ4XyJ0dm4+Fqz6pMjWNiaMPjlsOtWo/k1c6RpVw+2LIji8tlMIncLHcWdG/uSUHJOMIYAJFouJQi5FZcuXaK4uLjW3mnVSUpK0ocfoqKiDFonXNuiQUTkfkMikdCkSRN27drFmjVrMDMzq1NeWkZGBosXL67xWKtWrThy5IjeGALBA6XVag2+L0TuL8J6eZF0MQd7N3NijghFJheOpj5QBlF9ED+J9wtSKfxvD9gJvWW2yztRWg7bz1Xk9jQyFH/TasrJSUkmL6PmG7+5tYqR77Rk3KftGDKjSoCtpFBNcX6VvpC6VMP5CgPjVvtyVar6Vie8j0+t40e914o2g6+fSNywvRvNenjp85r8w4wTWdfNOcWBfy6Sl1FC0oUcTu+8yu4VgiFj5WCoqdJ2qD/27haozBSE9fQ0OJZ7vnb9Fa1Wa9QTqjaqJ09v2LBBr7gN9S+HFxG5F1RPvF6xYoWRSGpNVP/cA4SGhup7pvXr189AHLeSL7744qbCciJ3B0s7E0a904qeT4cw7tN2ACRdyOHklssPZdWgaBDdT5jawlOboOs7nGs+E4D1p4X+ZlKpjGFvf4Sjl49++IIXJ/Hr8xNJjK75Rm3taIalnQlugTZ0n1AVXqv0npSVlDPvxd169eXahLtuBRtnM6b82IWxn7TV7xv+Rgue/akr9m4WKFQymnStuXorqI0LlnaGRopnQztMLRXIlVJ6PFWlxHp2b9WT574/qnR+Br4YxtSfu+l/mveq0tlxC7TBzq1KvFBeboGsXPAASbRVztPK5pyV/c/i4+P57rvvDAoCqlOb4fTCCy9gb29f4zERkfsJe3t7gyq6xMTE644/evQo27Zt028PHDjQQPsIqLHCrLS0lBMnTtzaYkXuCpZ2Jrj4CWrWB/65SHxkJuVqDRlXa8/jfNAQDaL7DUtn6PwqPZsLLuo9MenklwhJbD5Nwxj31Q90nfA//XCdVsvK917l3L5d1502uI0rbYcI3pjK0FR8ZJXIo1wpxczq5sJlN0Imk2Jlb8qQGWEMmt4MZx8rpNKqBpgdhgfw9KwOtBsmJBqHdHRj6s/d6DGhkdFccqWM4a+34PG3WhLY4sYeLWvH2kNcUpmU0e+1ZurP3XAPskWCBJMioTrRXhKAq6vwe2V1TlxcHAUFBaxatYrs7GyWLVumn+vixYtER0eTnp5OZmamkacsICAAO7ubkwoQEbkXjBkzRq9XVL21h1qtpqioKr8vNzeXDRs26LefeeYZmjdvjlxumJHRqFEjZsyYwQsvvMD48eP1+yMjI436FdaF48eP8/HHH+uLFUTuPIEtq1Srz+1PYt8fF1j18VFijtaeFvEgIRpE9ykNnC0IcLKgTKNl2zlDwcAGbdobjd/4/SzO7Nx63TkrxbkSo7Mpyivj0vGqcFu3sbeuN3Ij3AJtaxQ0lMqkmFooadbdkwEvhNJ+eOB157FyMMXWxRypTMqA50NrHTewHmrSQa0FkU4/14ZMmfwck94YxpgxT/C///2Ppk2b4ubmhk6n48iRIwa9z/Ly8sjPz2fFihX8/vvv+j57fn5+PPnkk/pxvXv3rvNaRETuB2xsbBhRoUF27tw5dDode/bs4ZNPPuHLL7/U94KMjKyqGHVwcMDDo3a9LktLS+zs7PD19dX3JExOTuazzz6rtxDkunXrKC8vF9W17yIhndz0kidxpzL0nvmtC6I4vbPmRPoHCdEguk+RSCQ81kRov7GhImxWiYWtHVaOxt6RzT9/R25a7Za6lYMpTt6W6HRw/lAyCWeFUvK+U5oYWP73ColUglcjexSq2rvPX4tXiD39p4UikUCXMUE06uCGXCFl1Lut8KyHmnRwWxd6PNWInk+H4OLmhMpUgYWFhb4FSmUDzz179hicd/bsWaKiovQ5FpXu/+DgYAICAvQ5FA+SiJ+ISCUBAQEoFApycnI4d+4cO3ZUNWk+elTQCavU1zI1Na1zJSYIBQaVRQpardYgBF1cXMzp06dr9Rylp1c9zJWVlYkl/HcJmUxK/6k1Nx/euyoGTfnNdxG4HxDL7uvAXSm7r4GY1Hx6fbMHpUzK0Xd6YG1a1YsmOzmR8wf24BUSysr3X9Pv7/jEBFoNGl7rnCe3XObAPxeRK6WUl2mxdjRlzIdt9K7xBx2dVoemXItcWXejqi5UdqivRKFQ6L+slUolZWVlBuNnzJiBpeX11cFFRB4EVq1aVWteXGUZvUQi4ZVXXqlXQ2GACxcusHz5ckDoudayZUuWLl1q4C363//+Z9CbMS4uzqiarWfPnrRvb+w5F7kzrJsTweWoLKP9j01tik+T29Ps+HZRn/u36CG6j2ngbElgZdgsyjBsZuvqTttho3EPbsT4r37Q748+YCw+WB3/cMFTUV6mrdh2emiMIRC8TLfbGAIhfODn56ffHjt2rP73SmOoMm/I09NTNIZEHhoaNTLM5QsODtb/XllG7+vrW29jCCAwMJBJkyYBEB0dzerVq41CZ9d6ZTdu3Kj/vbIf5alTp+p9bZGbp9v4hviFOdKgtTMTvmiPX1hlV4IHu++ZaBDd5zzWtCJsFplc6xgHLx+enb8ciVRKWvwlspNrrgiJ2ruT3156Eiu7KuOqrk3/RGDEiBFMmDCBl19+GS8vL0aOHKk/5uXlRWCgkPt07Q1ERORB5loNopEjRxq1iakMKd8Mbm5u2NraolarycnJMTp+9epVfUharVbrc5dA+J+UyWSkpaUxc+ZMfv/99zpJBIjcGubWKvr+rwk9nwrB3FpF04pK4diIdP77OZKl7xzQdx94kBANovuc/hUG0d4L6eQW1V6JYWZljVdjIcE4+sBe1KUlRO3dibqsVD/m2PrVlJeWUl4iqFhbO5ri4CH2Faorpqam+Pj46N2ulQYQCDeNAQMGMHjw4AemA7qISF1QqVR6uYigoCAkEgkjRoww8AhV9xrVF4lEQuPGjY32BwcHo1AoyM/P13uiYmNjKS8vR6lU8uqrr2Jvb28QTouOjiY+Pv6m1yJyc7j6W6Myk6Mu1RAbkU5eRgkrPzrywGkViQbRfU6AkyXBLpaoNTo2n71+aWNQO0GBOvrgXnYt+ZX/fviarb98T7laTcLpCNLjBYHArMTzuPhcocuYoIcqXHa3kcvlDBs2jLCwMFq1aoWFhQXNmjVDJrv9ITsRkXvJmDFjaNOmDQMHDgSEarHp06cTEhJCt27dbipcVp2mTasSdX19fenduzeDBg3Se6cqy/4rxR+bNWumv2aPHj0M5rrV8FlZWRknTpzgzJkz/PHHHxQUPHiejruNVCbFK8RYY+16vSaroynXkn4l/54bUKJB9ABQ6SVadzrpuuMCW7ZDKpOTcSWB09s2AXBu3y6+e3IIf33yjsHY+JN/knBqA3kZD3bM917TpEkTBg0aVKMKr4jIw4KdnR19+vQxMHwUCgUjRoygU6dOtzy/o6Mj7u5CO4iOHTvStm1bTE1N9eHnqKgoNBoN0dGCAn11j5SPjw/Tpk3Te4rOnTtnVORQG7m5ufz999/6VjsAW7ZsYe3atfz1119ERUUxa9YsMjIyrjOLCNTcRaCyafiNOLj6En98cpTzB2tPDbkbiAbRA0D/poJA4IFLmWQUlNY6zsTCAhtnl+vOZW5jq//90D+rmD/1aYpyc27LOkVERERultGjR/PUU08ZFC8EBgbqy/7nzJlDUVERpqameHt7G5zr4ODA5MmTsbGxoaysTG843YjVq1cTGRmpF1nV6XQcO3bMaNwPP/wgGkU3wKeJA04+VrgF2tBropBTdulE2g29PlqNllPbrwCwY8n5646904gG0QOAj4M5Tdyt0Wh1/Hfm+mGztiOu3z162NsfGe2LPlRzV3gRERGRu4WFhYWRoaNUKvXeoNzcXEDIY6opLC2RSPSht7qEzTQajT7fqKCggLlz5/LJJ5/UOv6HH37gr7/+Yvv27fUWkXwUkCmkjHijBUNmNMe7iT0yhZTc9GIyrtYeclSXaVj3fdXfysbZjNLie/feigbRA8KAUMEdvP7U9cNmQW07Mvztj3nsxdeY9tsqWgwYqj/2/OI/cfTyYcaq9Qx69V39/piDokEkIiJyfxIaaqhGf72KtkqD6OLFi/z4448GKtrXcvnyZYPt1NRUI0OnurcKhH6Ge/fuZd8+8TvzeihN5HhX5BRdOl57Wsa+Py5w9bxQNegeZMMTM1ujMpXXOv5Oc++uLFIvHmvqxqcbz3M4Lotus3ZhZargj/+1RSk3tGklEgneTZvptzs9MQFrJxfcgxqiNKnq6xXQojVjPv2G5W+9xNXzZynIysTCTmw8KiIicn/h5+eHs7MzqampODk54e/vX+tYBwcH/dj09HT+/vtv0tLSMDU1JT8/n+TkZDQaDS1atGD16tW1zjN06FAcHR1xcXHhypUrLFy40OD42bNn6dKly+16iQ8lAS2ciI1I5/imBLKSC+k7pYm+iEen03HuQDJR+6oe8Jv38r7nRT6iQfSA4G5jSri3LccTsonNELrT772QTveG12+5IZFKadarX43HXPwDcWvQkKSYc8Qc2kfzfoNu+7pFREREbgWpVMqzzz5LWVkZEonEqHHytYSEhJCaWqW1tnevsVjtlStX9L8/8cQTXLhwQd+KpHnz5jRu3Fh/HS8vL15//XW+++47SkpKAKF1SF5e3l3tXPCg4RtapVgddyqD9Mv5OHkL71dsRDo7l1blC3k3scc92NZojruNGDJ7gBjQ1NVge/3pW8/IryzV37l4PjsXzTM6nnE5nvKb6EQtIiIicjtRKpUoFIobjmvbti3h4eF1ntff358OHTpgYmJCw4YNGThwoJHRZWpqyssvv8w777yjb147e/Zso7CbSBVyhYyQTu767ZjDVUZqzJGq34PbutB/aigy2b03R+79CkTqTL8mhgbRlrMpFJdpbmnOBq2r+v+c+G8tJdU0N2IO7WPxq9PYtXj+LV1DRERE5G6hUCgYMGAA7777Lp6engbHKpvJVvLUU08hk8mwtrbm1VdfZcSIEbXOq1QqkcvlNGzYUL9vyZIlt3fxDxkdhgcQ1EaofI7cfZXMxAL2/B5N7MmqcvzWA/1qO/2uIzZ3rQP3qrlrTXy//QIX0go4FJtJWn4pP41pbmQo1ZcVb88g+aJQpuoS0IChb8zE1NKKX6aMoyBbaODn3TSMYW9+gOQG7moRERGR+5nz58+zZ88eBg8ejJNT/VsX5ebm8s033+i3n3nmGb3XSMQYjUbLotf2U1JoHGl4bm7XO543JDZ3fYh5vnsgc0aHMbS58A/41eZofN7YgM8bG1h98upNzTno1Xdw9hPaUKRcjOHPj97m4rHDemMIIOH0SZJi7q1GhIiIiMitEhwczOTJk2/KGAKwtramf//++u1ff/2VCxcu3K7lPXTIZFKcfIwNkbBeXvc8ifpaRIPoAaWyDD+uIsEa4KVVNydZb25jy2MvvqrfTk+I49+vjPWKzh/YY7RPRERE5FGjRYsW9OrVS7+9devWGsdpNBqSkpLQ6XScPHmSjRs3otHcWprDg0h4X0N9KbdAG1oPuH9CZZWIBtEDSiNXK3zszYz2X0itW++Ya7F1caNx155G+4PadcK/RWtAyCnSPoL/zCIiIiLXUl0fKSsri9JS4y4Ce/bsYd68eezfv59///2XI0eOcObMmbu5zPsCtwAbRr3XClsXM8yslPSd0gSZ4v4zP+6/FYnUCYlEQruAqrJGU4Wg3Prr3jiKym6s9Hk1u4iTl7MN9vWe8iJdJ0w22Ndh1DgGvPQmJhaWFOXmcOVs7UJnIiIiIo8K5ubmvPvuu9jZ2VFeXs7GjRv1PdTi4+PZv38/p0+fBmDnzp3681avXl3n1iIPE/ZuFox8pxVjP26LifmNqwXvBWJSdR24n5Kqq5OWX8LU5ScYGOqGhYlcHzLrGOjA0omtaz1Pq9XR9etdJGQWsf75DjR2t9Yf0+l0FOXmoFCpUJpWeaC2zvuB09s30bhrL3pPeeHOvSgRERGRB4jt27frtY58fHwYNGgQ33333Q3PCw4OpmvXrjg7X19LTuTWuGtJ1ZUiVSL3BidLE/6c0o6xbX3o2aiqqeveCxkkZBbWeE5usZrGMzeTkFkEwMqjhjoaEokEcxtbA2MIILi90NH6wpH9aMpFXSIRERERMGwlEh8fXydjCIRqt7lz53Ls2LEbNkB9FLgfcqvqbRBptVo++ugj3N3dsbCwIDY2FoB3332XBQsW3PYFitQNC5Wcx1tUlX6ujai559nKI5cpqqZdtOVsKhrtjf8Z3RuGYG5rR2lhIfGnTt76gkVEREQeApydnXFzc6v1eKUWkqmpKW+88YZRY9r169cTFxd3R9d4PxMXF8fChQv56KOPWL58uT7seC+ot0H08ccfs2jRIr788kuUSqV+f+PGjfn1119v6+JE6sdHgxszrWsAAGsiEmt86th3McNgOy2/lIE/7COn6PofQqlURlCbDgBEi9VmIiIiIoDgVR8/fjwtW7Y0OiaXyxk6dCjdu3dn+PDhmJiYMGnSJBwdHQ3GRUVF3a3l3jdotVp2797N4sWL9Yrf8fHxBnbF3abeOUQBAQH88ssvdO/eHUtLS06dOoWfnx/nz5+nbdu2ZGdn33iSB4z7NYeoJvJL1IR/vI2yci0bXuhAiFtVflBukZrwj7dSrtUxo2cDEnOKWXlU6OnzWBNXfhzT/LpzJ8Wc5/d3XwHAN6wFg199F+k1TzsiIiIijyJarZYrV67g6upKcXExEomEsrIyHBwcahyflpbGTz/9BAgJ2i+//LKR9+hBR6PRcOHCBfLz8wkLC0Mur2qfevz4cdatW2cwPjg4mFGjRt3WNdzRHKLExEQCAgKM9mu1WtRiz6t7jqWJgh4NBcGxa8NmO6JTKdfqCHSy4PnugQxsVuXm3RCZfMPQmWtgkP73uJPHuBwZcfsWLiIiIvIAI5VK8fb2RqlUYm1tjZWVVa3GEICTkxPvvvsuZmZmFBYWcvHixbu4WoFt27axbNkyioqKiIyM5PDhw2i1WgDKyspISUmp95znz59nx44d/PPPP/z111+sXLmSDRs2sHv3bv3cAJGRhhXL5ubm9OtXcyPyu0W9u903atSIvXv34u1tKLT0119/ERYWdtsWJnLzDAx1Z2NkCmtPJfF6n2CkUkENdNMZ4cPdp7GQgN3a156hYe78czIRgMOxmQal/NcikUho3LUXZ3ZuAeDcvl34NKt7E0URERERkSpkMhmhoaEcPHiQiIgIgoKCbnxSPSkrKyM9PR13d3eD/bm5uezbtw+AL7/8Ur9/8+bNBoZL+/bt6dixIyYmJje8VmJiIitXrqzx2N69eykoKKBv375IpVKSkoQHdm9vb8aNG3dfeMfqbRC99957jB8/nsTERLRaLf/88w/R0dEsWbKE9evX34k1itSTLkGOWJrISc4t4Wh8Fq397Cku07A7Rmio1ztEMIhkUgmzRzZDpZDx+5HL/BuRdF2DCKDrhEkU5WYTe+IoUft20bzfIJz9jD2GNyI17hI2zi6ozMzr/wJFREREHhKaNWvGwYMHOXfuHEuWLGHEiBFGTWhvhc2bN3P8+HGGDBmiF5PMyspizpw5NY6vbgwB7N+/n/T0dJ544okbXismJua6x0+ePMnJk1VFOebm5owfPx7pfdIjs96rGDRoEOvWrWPbtm2Ym5vz3nvvce7cOdatW0fPnsZKxyJ3HxOFjL4VXqA1EYksPhDPz7svUaLW4m5jSoibYRx1UEXobOOZZErLr1/6qDQxZfBr72Ht7AI6HcvenE7C6Yh6rS8hMoJlb7zIhu++vPFgERERkYeY6lVqsbGx7Nix47bMq9PpWLduHcePHwcEQcjU1FQA/b66EhMTQ25urtH811JTTzeFQoGFhUWN8wYGBt43xhDU00NUXl7Op59+ytNPP11r7xaR+4NBzdz549hVfj9yxWB/7xAXo4Z6rXzscLEyISWvhF3R6XoPUm1IJBICWrbl+PrVAOxbuRjvps3QlJeTHh9LUsw5Alq1xcrBsHliaVERF48eJOG08IQQF3GcvIx0rBwcja4hIiIi8qgQEhKiDyGdP3+efv363XLj08jISCPD5+jRo/Tv39+gzL9fv37odDoaNmzInj17OHbsGA4ODnTt2pVjx47px54+fZrWrVsjlUpJSEhg6dKlAIwbNw4/Pz/y8/P1r2HGjBmUlZWxceNGOnTogLW1NcuWLSMrK8tgPYGBgbf0Gm839a4ys7Cw4MyZM/j4+NyhJd1/PEhVZpVotDrafLad9HzD/jp//K8trXztjMZ/uvEc8/bE1qnaDCA7JYmFLwptPuQKJU998zPzpz2tP27r6sbT384zOGfTT99wdvd2g30dn5hAq0HD6/y6RERERB42ysrK+PPPP/UelqeffhovL6+bni8qKoo//vhDv21vb09mZiYqlQpPT099AveLL76Ira1trfNoNBqOHDnC5s2bMTExQSKRYG9vT2pqqr6IysrKilGjRjFvnvB97+rqyv/+9z+jubRaLZGRkZSXl+ury95444065SbdCne0yqx79+7s3r37phcncneQSSX0bGQsCR/uXfOHf2Co4LLddi6V/JIbVwvaurjxv5+XYGHvQLm6jI0/fG1wPDs5iczEK2SnCE8Mh1f/YWQMAZzft+uG1xIRERF5mFEqlYwZM4amTZsCt65L9N9//+l/9/f3Z8qUKVhaWlJaWmpQzXY9YwiEpO/KcvmSkhKKi4u5evWqQUV5Xl6e3hgCak0Ml0qlhIaGEhYWRqdOnRg4cOAdN4bqS72Tqvv27csbb7xBZGQk4eHhmJsbJsUOHDjwti1O5NZ4opUXKw4LglcKmYRpXQORSWt2w4a4WeHvaM6l9EK2nE1lWLhHjeOqY2FrR2j3Puz/YxmJ588aHV/08rMAdJ0wmX0rlxgcs3J0pjA7k/TL8aRfjsfRy6eer05ERETk4aJhw4acPn2aqKgoFAoFdnZ2WFpa4uLiUmsezrUUFBSQn58PCEbI8OHDUSgUhIaG6qvKQHBu1AUTExOCgoI4e9b4O97S0lJ/rUqaN79+hEEqldKtW7c6XftuU++Q2fUSoCQSyX3Rj+R28yCGzCo5eTkbR0sVbtamSCRcNy49Z/sFZm+NuWFz2OrkpKaw4IVn9Nsj3v2U/Mx0Nv30Ta3nSCRSRrz7MSf+W8vFo4doOWg4nZ6YUOfXJCIiIvIwUlZWxldffWWk6RcYGMiYMWOue25RUREHDx4kLS2N6Oho3NzcmDx5sv54eno6P/74IwDh4eH069evzqXu58+fNyqn79q1K0VFRRw+fFi/70YhuHtBfe7f9fYQXVuSJ3J/E+ZV9w/nwFA3Zm+NYf/FDNLzS3G0VKHT6XhnzRkUMinvD2hkZFDZOLvg2iCY5Jjz2Ll74tW4KaVFRYCxQWRhZ8/YL+agUKlQqEwozs/j4tFDnN+3m46jxiG5j6oNRERERO42SqWSoKAgzpw5Y7D/woULrFu3DisrKzp37lzjuWvWrDEoew8ODjY47ujoSJs2bSgoKKiXMQRChwpvb29MTEzo0aMHlpaWmJiYkJGRoTeIOnXqdN8ZQ/Wl3gaRyMOLj4M5oZ42nLqSw4bTSUxo78upq7ksrwi77b+YwR//a4utuWGvmfB+g1kf8znNej8GgMrMDHsPLzKvXjYYN+T19zGzqmol4te8FUpTM/Iz05k9eiAT5/yKjfP1K9xEREREHmaaNGliZBBBVal8o0aNjHqhabVaIw2ghg0bGs3Rp0+fm1qTXC7nqaeeMtrv4ODASy+9hEqluu/ygW6Gm3ok3717NwMGDCAgIICAgAAGDhzI3r17b/faRO4BgyqSq/89JSRDbz5bJd1+Ia2AsI+2cjzBsHQyqG0Hnl/8J816PVY1z6vv0LBDF1oPeRwQcoYcvX0NzpMrlQS2aqffXvDCM1w8dhiRu0dRXi7/zvqYr0f2Z+H0yRRkZd7rJYmIPNL4+/vj7u6Og4MDHh7GuZwnTpzQ/67VasnLyzMoowdB/flao+lOYW1t/VAYQ3ATOUTLli3jqaeeYujQobRv3x4QlCxXr17NokWL6qRm+aDxIOcQ1Ze0/BLafLodrQ72vNqVCYuOEJteaDQu7rO66WRotRqOrVuNR8MQ3BoYP7HERxzn78/eN9j38sp1N5z76vmzHFu3mm4TJmPl6HTdsSK1s/qLD4g9cVS/HdK5B32em37vFiQiImJAdHQ0v//+u37bzMyMl19+GbVazRdffGEwtkmTJvTv3x+ZTGbQSPVR5o6W3X/yySd8+eWXrFq1ihdeeIEXXniBVatW8fnnn/PRRx/d9KJF7g+cLE1oX9G+Y+S8g8SmFyKRQO8QwxL+s0l5dZpPKpXRatDwGo0hAO/Q5nQZN8lgX1LM+RvO+8cHb3Lp2CHmT3ua+FMn+Puz90mMPlenNYkIaDUaA2MIIOFMRI0KtCIiIvcGX98qz7qpqSlFRUWcOHGCb7/91mhscHAwKpVKNIZuknobRLGxsQwYMMBo/8CBA43cdreD/Px8pk+fjre3N6amprRr146jR6u+xAsKCpg2bRoeHh6YmprSqFEjfv75Z4M5SkpKmDp1Kvb29lhYWDBs2DC9hLmIMZWaRMm5JQB0buDIL2Nb8MHAEP2YfyMSb8u1JBIJ4Y8NYsjrVV6i8/uvr3NVmJONrlpy/9+fvkd8xHFWvvcq344ZzOltm+q9jsgdW1jz1UeUFBQAgvDkHx++xZZf5qAuK73B2Q8m1aUS5EoVAAWZGSSJhqWIyH2DUqnkf//7HxMnTiQ8XGimvXHjRkpLjb+XAgLq31dSpIp6G0Senp5s324ssLdt2zY8PT1vy6Kq88wzz7B161aWLl1KZGQkvXr1okePHiQmCjfkl19+mU2bNrFs2TLOnTvH9OnTmTZtGmvXrtXP8dJLL7Fu3Tr+/PNPdu/eTVJSEkOHDr3ta31Y6N3YMLG5U6AQix7fzodfxgr/kGtPJaHR3j5Pgl/zlgx9YyYAMYf2ob2OfMOFIwdrPaYpL2fr/B8oLTIO8wGc3LSOc9cYXJlXr7DllzlcOnaYw2sEddc/P3qbK2dPE7ljC6e2bKznq3kwqMzXCuncnReX/k1IZ0GX5Ny+nfdyWSIiItfg6uqKp6cnYWFhRsfatWuHRCKhadOmqFSqe7C6h4d6G0QzZszghRde4Nlnn2Xp0qUsXbqUKVOmMH36dF555ZXburji4mL+/vtvvvzySzp16kRAQAAzZ84kICCAuXPnAnDgwAHGjx9Ply5d8PHxYfLkyYSGhnLkyBEAcnNzWbBgAbNnz6Zbt26Eh4fz22+/ceDAAQ4dOnRb1/uwYGWiYEI7HwBszBQGIo1dghyxMpGTmlfK4bjbm4Dr1aQZJpZWFOXmcPnMqVrHXTi8HwDL6/RAu3D4gNG+1NiL7PjtF/77/msKc7IByMtIY9GMZ/Vjjq37h69H9ic/I12/b9/KJfw762Oi9tyepov3AzqdjotHhc+/f8s2ADTs0BWA6AN70ZQLOiiacjWRO7dQnF+3EKmIiMidw97eHlNTU/22n58fPXr04MUXX6wxciNSP+ptED377LOsXLmSyMhIpk+fzvTp0zlz5gyrVq2qsX/JrVBeXo5GozHKYDc1NdUrbrZr1461a9eSmJiITqdj586dxMTE0KtXL0AoVVSr1fTo0UN/fnBwMF5eXhw8WLun4VHn/QGNiPusHxHv9cLaVKHfr5LL6NfEFYB/Tybd1mvK5HKC2giJ+uf376lxTFFeLleiIgF4/N1Pefy9T5FIpThco3QdtVfwclw9d4aspKsAXDgiGEk6nZbog0JVZMzBfdwIjVrNxaOH+O/H2RTl5tT7dd2PZFyOJy89FblCiU8T4anTs3ETzG3tKCksIO6kUOJ76J8/2PLzHFa+9xpFebnXm9KAkoIC9q5YRG6aGJoWEbmdDBo0CIAWLVowbtw4pFIpNjY2KBSKG5wpciNuqux+yJAh7Nu3j8zMTDIzM9m3b5/+j3Q7sbS0pG3btnz00UckJSWh0WhYtmwZBw8eJDk5GYDvv/+eRo0a4eHhgVKppE+fPvz444906tQJgJSUFJRKJTY2NgZzOzs7k5KScu0lASgtLSUvL8/g51FDIpHUWuk1qJk7ABvPJFNafnuVyYPbC6JjF44coLysDBC8GQf+XMGZXdu4dOwwOq0WR29fbFxc8QxpynPzVzD2i++YsWo9k35YCMCVqEhiTx5l1cw3+O2lKWz8fhaHV1c1Ozy/fzfqkpLren3ajxxrJBZ5/sDDIS8RU+Fl82raDEXFA4dUKtO////O+pjysjIOr14FQFbSVX59/hnK1TfucwdwaPUqjvz7Fzt++/nGg0VEROpMcHAwL7744k1rConUTr0NoqNHjxpIdVdy+PBhjh07dlsWVZ2lS5ei0+lwd3dHpVIxZ84cRo8erW8h8v3333Po0CHWrl3L8ePH+frrr5k6dSrbtm276Wt+9tlnWFtb63/uRG7Ug0xrXztcrEzILyln5/l0CkrL0d6mfCL3oEZY2DtQVlxE3Enh83Q1KpKDf61gyy9zOL1dSJgObF2lX2RiYYFUKqiuWjk64dGoMeh0/PdjlVr2uWpNZCUSKckXopkzfjjpl+MBeOb7BTz52bdQYQS2HT6algOHMfjVdw3W9yDn11w6foTvJzzOhSMHiK4w7ILadjQY07BDF/3v2xb8ZJC8ri4pJu6kYVVabcQeF74j4iKO68OTIiIitwdbW1uxkuwOUG+DaOrUqVy5csVof2JiIlOnTr0ti6qOv78/u3fvpqCggCtXrnDkyBHUajV+fn4UFxfz1ltvMXv2bAYMGEDTpk2ZNm0aI0eOZNasWQC4uLhQVlZGTk6Owbypqam4uNSsivzmm2+Sm5ur/6np9T7KSKUSBjYTKtHeXh1J05mb+WpL9G2ZWyKVEtxO8O6tnf0pX4/szx8fvQ2ATqsl5aKgxtqgdfta52jUUWgcWFJD3outqxteTUKN9ls7OePsF8CMleuYsWo97UaMQSaX49e8JS+vXMeUX5YikUpJuRhDdnIi6pISrkRFcm7fLi6fOcXuZQvZtmCugQFxv7Hum88oKy5i7defkp2ciEyhwD/csGedk48f7sGNADi7y/ihImrPjQ3CrKSrZCcL4VSdVsv5/XuIObSPvIy02/AqRERE7gY6nY7dyxby9cj+HPhzBSWFBXfsWukJcWz79cd6heXvBPU2MaOiomrsZhsWFkZUVNRtWVRNmJubY25uTnZ2Nps3b+bLL79ErVajVquNGs7KZDJ9z7Xw8HAUCgXbt29n2LBhgCB0dfnyZdq2bVvjtVQqlZitfwMGNXNj3p5YMguFsNbcXZd4sXsgJoq698epjeB2nTi27p+qHdfo4ti5e2Lv4VXr+Q3atGfHwp8pV5cZ7DextGLwa++RFHOehNMn9ft7Tp523fVIJBLMbWzxaRpGXMRx1nz1MVmJNRvJJuYWFOXl0GXsRJSmZtedtz5kXI7nzK5tqEtKaDloeL1bnJQUFKC5Jtxl7+6FysxwjRKJhGFvf8TPk5+krLgYgC7jJuHVuClLXnueuJNHKS7Ix9TCstZrXaqoXpPJ5WjKy9m1ZD4gGJ3PfL+gXusWERG5N1w6dlj/PXzwrxUc/fcvRrz3Sa2acnXlyL9/IZFKaTmgqtL770/fozAnm8KcHAa98vYtzX8r1NtDpFKpatTwSU5OviMuvM2bN7Np0ybi4uLYunUrXbt2JTg4mKeeekrf6O7VV19l165dxMXFsWjRIpYsWcKQIUMAQVZ84sSJvPzyy+zcuZPjx4/z1FNP0bZtW9q0aXPb1/uo0MjVCj8Hc4N9287dngRaJ1//GvdLZcLnq0G1cFlNqMzM8WsheD4CWrbhhSV/8dLv/zL11xXYuXkQ2KrKEO4ybhJNu9ctFl8ZTqrNGAI4vHoVkds3c6havhKAuqz0pr1H+ZkZLH51Gsc3rOH09k0seOGZGqvorkdsDaGudo/X3D1boVQRWM0DF9CyDY7evjh6+aApL79hIvql40KFZ6vBj+v/ZgC5aan6EGUlOq2WjMvxohikiMg9oKSwAK3WOA80PyuDf2d9bLCvXF3G7+++yp8fvWXUp7KuxJ48yt4Vi9izbCEZVxKENRQU6MPqF48erHE9d4t6G0S9evXSh5QqycnJ4a233qJnz563dXEglM1PnTqV4OBgxo0bR4cOHdi8ebM+o37lypW0bNmSMWPG0KhRIz7//HM++eQTpkyZop/jm2++oX///gwbNoxOnTrh4uLCP//8U9slReqARCKhUwPDsvfVJ26fWOPImZ/j3bRKc8M1MIhmvfphZm1DSJcbf846jh5Pww5d6DB6PAqViT7HCASDqev4SQS2akfT7r3rvK6AljV7FGvi+Po1pMZeZO+KRWz79Ud+fGoku5b8WufzQTCi9q1cysG/VhgdWzv7U1JjLxqOLy2pda5LFSX2DTt0Ibz/EEa+/zn+4a1qHd+kW2+QSHALaoS1k6BS3rCTEIqsrOC7lvKyMi4cOaAXfGzcpQc+oYa6KWd3bUNTXs6+lUs4vPoPZo8eqDf2RERE7h5JMef56Zkn2PHbPKNjm376ttbzLp85zeJXptX7AS/hdASrP/9Av31m51Z2L1vI/GlP6/e1HDRcX0xzL6h3L7PExEQ6depEZmamXiQqIiICZ2dntm7d+lAmID9KvczqQ2ZBKdNXRRDoZMnC/XHIpRIOv9Ude4vbF24szs/j2PrVhPbsh9V1dIfuFktee570BEGRfeKcXzG3tWXNFx+SnhBXJ62eqQtXYmJuUadr7Vu5VF/lVRN+zVvi4OWDras7m+d+C0Cf517SCyxWUl5Wxk/PPIG6tIQxn8zGJaBBna6fFh+LhZ09ZlbWABRkZTLvuafQ6bRMnPOrPmwXH3GclEsX2P/HMv259h5eTPj6J87t3cnGH76u0/Xq0sOuOjqdrl7jRUREqtj260+c2roRmULBlJ+XYmIhfC9pysv5dsxg/bjnF/1B9KF9bPl5jsH5I979BCcffxKjo/Br3pKTm9ZRmJ1Fh9Hjjf4vNeXlLH7lOX1uYU20HvI4HUaNu30vsIL63L/rHeNyd3fn9OnTLF++nFOnTmFqaspTTz3F6NGjRR2ERwx7CxVLJwqhqWMJWZy+msu6U0lMaO97gzPrjqmlFR1Hj79t890qfafNYP23X9Bm6Ei9QTDs7Q9BB8fWr2b/qqXXVdmOPrCXxl17IpFKDLxWlZQWFRK5YwuNu/Tk4lFDnaynvvmFsuIilr/1EgCxJ44a9SLb9NM3hHTuTsaVBEoK8vFo2JjLZ0+hLi3Bws4eZ7+6S/s7+fgZbFvY2ePVJJSE0yc5t3cnbYePRlOuNmrOC4JHDwQZhcLcHBw8vGocV53E82fxaNj4huvSajQsf+tltJpyxnz2LXLxe0dEpF7odDr9d4dGrSb64B5Ce/YDIPF8VS7wgJffRGlqRpOuvQhs2Y7dyxZyZucWANZ89THqEiHPsPWQx/WyJn7NW+kLMyrZOv8HA2NIIpGi0xl6mPxbGBZ43AtuSofI3NycyZMn8+OPPzJr1izGjRsnGkOPOIMrtIlWn7w9YbP7FUcvH56aPdegPF0qlSGVyWg5cBgvLvuHQa+8U+v52379kW/HDGbNFx/WeHzHwp/ZvXQB/3z+vl5QshI7N3dc/AOZvnz1ddeYGnuR5W++xKqZb3Bq60YuVrQ68Q9vbaSrVF8adRTUrKP27kCn03HlzOkax7UdPhoQqgZb9B+CT7PwWnO1PBoJRtDZ3cYtgWoi6cJ50uIvkXElgdgTR+r7EkREHmm0Wg1/fvQ2+ZlVavxnd1X971X+T4V07m5QzWtiYUGvydMY/o6QW1RpDAEGGm9xEcfJTk5k/rSJHFv3DymXLhhUrE5fvoaAlob5u60GDcfFv26e6ztJvb8dFy9ezIYNG/Tbr732GjY2NrRr146EhITbujiRB4eBzdyQSSWcuprLpfQ7V555PyORCF6fgJZtGPfVD5hZ22BiYYlX46a0HDTcYGxcxHF9UmElOp1On5+TfCHaIEbfdfwk/e8yuYIWA4x78ZlZ2wCC0VVZYbft15+I3CE80V37JXQzBLRqi1ylIiclmeQL0Vw8Ztj+ZuCMt5ixaj1WDk5G53adMJnwx6oEXKUyOa0Gj6D9iCcBoYfd9fKgKqnuFVs3+zMO/Ln8Zl+OiMhDi06nI+ViDGd2buXHiaNZ8e4rQluk0xFcOSs8yFg6OCKVyUi+GM3sUQP565N39caLXw05hhKpFM9GTa573XP7dnF6+2by0lPZvWyh3qMN0GrwCGRyOc16PwaAnZsHM1atp+MTE+6L8He9c4iCgoKYO3cu3bp14+DBg3Tv3p1vv/2W9evXI5fLH8pkZTGHqG489dsRdkanM61rAK/0DrqluXQ6HRsjU2jqYY2n3e0rX7+blBYVIZGgL7+fP20ieelVlXgtBw2n0xMT0JSXI5FIyLiSwNLXXzCYo83QkbQa8jhyhdLgC0NTXk5a/CUAVr3/OqG9HsOzUROjypBKlKamPPfrCmTyW/fkbvzha87t3Yl30zASIiNAp8PC1g6XgAb0n/4GshtUmybFnMfUygobJxckUik6rZYF0yeTm5pCv2kzaFjhhQIoV6uRyWRIpFIunznFnx/VXJL77LxleoNQREQEIndsYcsvhnk/Tbr3Rq5UcvK/dQD0eOY54iKO66UyKpHK5ExdsKJW6ZB1sz8j5vB+LB0csXZy5mrUmRuuxy+8FUNee0+/fSUqEmtHZ6wcjR+ebid3NIfoypUrBAQIeQhr1qxh+PDhTJ48mfbt29OlS5ebWrDIw8GQ5h7sjE5n9clEXu7ZAKn05i3+jZEpTF1xAhcrEw691f3GJ9yHXKvxM+ytD9i1eD5FeXmkxl7g3L5dtBwwlCWvv4CVgxM+TY07WfuHt0ahNE5Sl8nluAYIRufUhSuRK5RotRpMLK1qFKS0cXa7LcYQCGGzc3t36rWcFCoTJn6/oM65PG4Ngg22JVIpjTp24+BfKzizezsNO3altKiQ316aQmFONk2696bN0JG1GkMAcyc/ycj3P8fJzx+pTC7mFYk88uxeaqz5Fbl9s/73oLYdadq9D6ZW1kYGkWdIk+vqqPWc/Dy2bu406dYbUysrMi7HY2JhxbH1/xhcozpdxj1jeI0beJruBfUOmVlYWJCZKXQ537Jli77U3sTEhOLi4uudKvKQ06uRMxYqOYk5xaw7nYTmJtt5HIrNZOqKEwCk5JVwPuXh6CVn5+bB0Dc/YNSHX6IyN6cgM4Nfn59IQWYGSdFR+tBP835CWMna2aVOSdAKlQkSqRSZXKFvxSGTy5n80yJ9xVnHJ25fYvq1St8W9g63bICEdBZK+i+fOUV2ciIb5nyl1yaJ3L6Z+VOfNhhvYWfP2C/mIKt23VUfvMH340fw54dvibpGIo80ZSXFqEtLrzum3wuvIJFK8Q9vhW+zcINj7kGNajlLwMTCgg6jxmHt5IzSxBS3Bg2xc3OncTVJFBf/QP3vE+f8iq2L2028krtLvT1EPXv25JlnniEsLIyYmBj69RMy08+ePYuPj8/tXt8DhUajQV3H5pcPK4+HObP5bApfbogkIS2XyZ38bnxSNXQ6HTN+P4q7ZVUF1tbIq/jY1G+e+52GnXsSc0gQOJSbGgpchvYdSGC7TpiYW1BaT02OkK69iD19Er+wFijMLeg0fjIth43G3NqGkpIb5+fUlfZjntK73TuNm3TLc6usbPBt2ZbUSxdY+aHgCTKzc6hxrE+zcJr3HYiVkzMTvpvP7++8YnA8Oz2NxJjzeATdmqKuiMiDSE5KMlfORaLVlGPt7MLE7+YjkUj457P3iYs4DoBvWAt9latMrmDom4I+0I7ffiH+1AlCe/a9qWu7NQhmyBvvI0GCR0gTEqPOoFaX1VtZ/15R7xyinJwc3nnnHa5cucKzzz6r77j7/vvvo1Qqefvteye7fae4UQxSp9ORkpJi1C/tUaRUrSG9oOom7m5jUudkOZ1O8Ahd61mSSyU4W5lwH+Tc3TbK1WqKamh6KpXJsLCzv6W5dTodEuCOvmE6XYWirASp7NbbtYDwVFuSn2+wTyqTGcgYmFnbIFcqDc8rLqKkoCqRX4cObXk5iadPEBjSmGYV5cQiIg87OakpLJ7xnL6oolnvx+j+9LOAoD696cfZgIR+z7+CuY3tPVzp3eOO5hDZ2Njwww8/GO3/4IMPahj9aFBpDDk5OWFmZnZfZMvfK3Q6HaqMIsoqbmKO1iZYmipvcJZAUVk56qwi/baHrRlJOcVodTpc7MwwUz483Z11Oh05aSmUl5aiMjNDKpWh0WiwsLE1CAM9Smi1GjKvVrVFMTE3x8LOAXVJMTKlEqlUVuv/VrlaTVFuNjotlJUUodZoUKjaceHoAUwtLPWhRBGRh5noA3sMejj6Nmuh/93SzoER7356L5b1wPDw3GHuERqNRm8M2dvf2pP9w0KAm4rE7GLyStRkl0nIKCnD2lSBm43pdc/LLi1GIheMJ3dbU+zNVRRppGQXlVGskWJnYnI3ln/XMPH0RgdGzYkfZeRePmjK1ahMzfSeJ1PT639uADAxwcLSEp1OR2rsRRQyGdjZom7anP9++gb34BAsbO3u8OpFRO4tcRHHDLY9Q+6/xOX7GfGb+BapzBkyM3swS8PvBAqZFBdrwXgpUWtQa7RkFJRSrtFSrtGSV6yuMek1r7gcAFdrE+zMBMPIxkzwluQUq9E+ZImyEqlUNIauQWligqmF5U2H4SQSCfYenphaWqGQyZDK5SjNLTm3b9ftXaiIyH1E/KkT7P19sYHKdJPuvVGoHq6HyDuN6CG6TTzKYbKaMFHIMFXKKC6ryv/IKVZTWFpObrEaU4UMLzszVAoZOp2OsnItpeUaJEiwM6/S3LFQyZFLpZRrtRSUlGNl+miGk0TqjkJlgrWTCTJTM5JSUpFIJETt3k6L/kPE/1ORhw6tRsPfn1bp+zj6+DH2s29vWZX+UUR8x0TuGDbXGC9JOcXkFgsetWK1hovpBZRrtJxPySc6VUimNVfJkFX7R5ZIJFVeoqJHu4JPpH7IFQos7R2QyuVkXEkgLe7SvV6SiMhtQ6vRsO3Xn/jmiUEG+29Hi55HlZt+19LT09m3bx/79u0jPT39xieI3Hd06dKF6dOnA+Dj48O33357W+e3NVNiopBhZVKzV0ej1ZGaV4JaU9WioiYPUFXYrIzE7GLS80vvms5MfHw8EomEiIiIW55r0aJF2NjY6LdnzpxJs2bNbnlekdqRSKV4NRZ0k+raK01E5EHg0rHDnNq60Wi/fw0tN0TqRr0NosLCQp5++mnc3Nzo1KkTnTp1ws3NjYkTJ1JUVHTjCUQeGeQyKQ2cLfFxMEcpr/mjllloqLNjZWIcxT28fy8vPTOG7uHBBLg70LFNC379bYn+eLlWW6OBtGfPHgYMGICbmxsSiYQ1a9YYjZk5cybBwcGYm5tja2tLjx49OHz4sNG4O8Err7zC9u13/iZd22t/VAhs1Q6Ac/t3oykXvYwiDwfRB/cabLsENKDD6PF1EnMVqZl6G0Qvv/wyu3fvZu3ateTk5JCTk8O///7L7t27mTFjxp1Yo8hDgLedOQqZFFOFjEauVvjYmxuNMVPKUcqNk2kPHjxIkyZN+PqXxfy1ZR+DHh/DlElPs3DFXyTnFnMuKY+EzCKKy8qJzygkPb8EnU5HQUEBoaGh/Pjjj7Wuq0GDBvzwww9ERkayb98+fHx86NWr113xelpYWFy3MrGsnqKMIjXjFhSMua0dJfl5Bo1hRUTuZ4rycrl0/EiND3ua8nLiTwlq/qaWVrQd/gSjP/qK1oNHiHlyt0C9DaK///6bBQsW0LdvX6ysrLCysqJfv37Mnz+fv/76606sUeQeMHv2bJo0aYK5uTmenp4899xzFFQTv6sM/6xfv56goCDMzMwYPnw4RUVFLF68GB8fH2xtbXnhhRfQaDSYKmU0dLXi0JY1tGndCncnO7o1D+KNac+Qn51JgKMF3vY1V+q99dZbfPnZpzRr0RpPH1/GTJxC+y7dWfvvGiF8BuSVqLmQVkBeiZrk3BJiUgvwa96BDz78kCFDhtT6Op944gl69OiBn58fISEhzJ49m7y8PE6fPm0w7vz587Rr1w4TExMaN27M7t27b/geLlq0CC8vL8zMzBgyZIi+5U0l14bMJkyYwODBg/nkk09wc3MjKEjoVXblyhUef/xxbGxssLOzY9CgQcTHxxvMtXDhQkJCQlCpVLi6ujJt2jQAvXr8kCFCQvGjqCYvlcpoVNEwVgybiTworP/mc9Z8+SGzRw0gNy3F4FjM4f2UFhViamnFlHlLaTfiCb3ytMjNU2+DqKioCGdnZ6P9Tk5OYsisAp1OR1FZ+T35uV25NVKplDlz5nD27FkWL17Mjh07eO211wzGFBUVMWfOHFauXMmmTZvYtWsXQ4YMYePGjWzcuJGlS5fyyy+/GBjKarWajz76iFOnTrF4xR8kXb3MezOew0wlRyGr/eOolEtp6mFDE3drAPLz87C+jtJqabmGsnKtPom7LpSVlTFv3jysra0JDTXs1/Xqq68yY8YMTp48Sdu2bRkwYICRgVOdw4cPM3HiRKZNm0ZERARdu3bl449r7kRfne3btxMdHc3WrVtZv349arWa3r17Y2lpyd69e9m/fz8WFhb06dNH70GaO3cuU6dOZfLkyURGRrJ27Vp9A+ajRwWPyG+//UZycrJ++1Gjsqdb3MljFOXm3NvFiIjcgNKiIq5EReq3f33+GSI2b6C0qIgfn3mCjXO+AgxbcIjcOvUuu2/bti3vv/8+S5YswaRCKK+4uJgPPviAtm3b3vYFPogUqzU0eq/mjr93mqgPe98WRefKZGsQvAwff/wxU6ZM4aefftLvV6vVzJ07F39/fwCGDx/O0qVLSU1NxcLCgkaNGtG1a1d27tzJyJEjAXj66aomnb6+vph/O4euHdtSUFCAhYXFDdclkUg4tmMjZ0+d5N3Pvrnh+KvZxVzNFpoO12QsZuSXsmXTRiY/NZaioiJcXV3ZunUrDg4OBueMnTiZoUOHIpFImDt3Lps2bWLBggVGRmIl3333HX369NEfb9CgAQcOHGDTpk3XXa+5uTm//voryor2FMuWLUOr1fLrr7/qXeG//fYbNjY27Nq1i169evHxxx8zY8YMXnzxRf08LVu2BMDR0REQFOZdXB6MfkJ3AnsPL1z8A0m5dIFz+3YT/tigG59UT8qKi1j1wZvYurrz2AuvsvbrTygpKGDY2x/dcvPbR4WoPTtIS4ij4+jxyOSPlirM5TOniT1xhPYjnyT+1HGj49sXzqUoL5eS/Kpm175hLYzGidw89fYQfffdd+zfvx8PDw+6d+9O9+7d8fT05MCBA3z33Xd3Yo0i94Bt27bRvXt33N3dsbS0ZOzYsWRmZhp4Ac3MzPTGEICzszM+Pj4Gho2zszNpaWn67ePHjzNgwAC8vLywsrLisd7Ck/vly5cBCAkJwcLCAgsLC/r2NW4wuHPnTp5/dhLzfvmFgV1aY6GSk3D2OO2CPWgTJPxsWP1Hja/pclYRao2W3KIyEjILKSotJym3GL/QVhw+epwDBw7Qp08fHn/8cf2ay8qFCrjAxs2JTMwlKjkPnURKeHg4pyLPotPpalzzuXPnaN26tcH16/LA0KRJE70xBHDq1CkuXryIpaWl/hp2dnaUlJRw6dIl0tLSSEpKonv37jec+1GnUYWX6Oye2x820+l0nNy8gbS4S0Qf2MO5fbu4ePQQV8+d4eLRg2z+eQ4/TxlHfmbGbb/2/YhWq6G8njlw6rJS/vtxNsfXr+bbMYNJjb14h1Z3/1FSUMCfH73F8Q1rOLz6Ty4ePQRg1HLm5Ob1Btt+okF0W6m3Cd64cWMuXLjA8uXLOX/+PACjR49mzJgxdZPYfwQwVciI+rD3Pbv2rRIfH0///v159tln+eSTT7Czs2Pfvn1MnDiRsrIyvSq34pqnXolEUuM+rVYwKgoLC+nduze9e/dm+fLlODo6cvnyZXr37q0P/2zcuFGv/n3t52n37t0MGDCAb775hgkTJgDg52iBa6d2REREoNEKTU11JlZIFDJK1BqD83U6yCosI6uwDLVGqxeNNDMzx87CjpCGQbRp04bAwEB+/Hkeb7z5Jvml5QZzlGu0pOaVUKLWoistJy2/9Lprri/m5obJ5gUFBYSHh7N8+XKjsY6OjqLSdT0IbteJ3Ut+JT0+lrT4WJx8/MhNS+XImj9pNXgE1k7GqQAAKZcusH3BTxRkZTL49fdx9vU3GnN21zb2/b5Yv/3fD1/rf9/w3Zf63+c9N4Ghb36Ab7Pw2/jK7j/+nfUJSeejGPPZt5Tk51GQnYVcpcKnaVit51w5a5i39+fHbzNt4ao7vdR7TklBAT9OHKXfPry66jWH9RlAaK9+/PPZTMrLSvXeoXYjxtB2+Oi7vtaHnZvySZqZmTFp0qTbvZaHBolE8kA3Ij1+/DharZavv/5af8P944+avS714fz582RmZvL555/j6ekJwLFjhr13vL29azx3165d9O/fny+++ILJkycbHDM1NdXnzFSnWK0hJbfEwDDKKChFoxXCYGXV9I/SC0oxU8nILlRToi4nLaeAC6n5pOeXAnD65DHC27QHIC23iFMRJxg1YRKpeSU08fIyquxo2LChUfn+oUOHan9zaqF58+asWrUKJyenWjs1+/j4sH37drp27VrjcYVCgUajqfHYo4SppRV+4a24cPgAS19/gcff/4yDf67gSlQkidFRTPj6J6NzLhw9yNpZn+i3183+lGe+X2A0bssv39d5Hf989j4zVq2/8cAHhPysDEwsLFEoVQBciYok9vgRABa88IzB2MGvvVerTs6FwwcNtksLC8lNS63VUH3Q2bVkPhePHsI1MLjWMa4NgpBKZYx8/zOWv/0yANZOzrQZNqrWc0RunjrdtdeuXVvnCQcOHHjTixG5PwgICECtVvP9998zYMAA9u/fz88//3zL83p5eaFUKvn++++ZMmUKZ86c4aOPPrrheTt37qR///68+OKLDBs2jJQUoeJCqVRiZ1d7w05NaTG5Vy/q84ASryRwNvI01jY2uLp7UlRUyK9zvqZrr37YOzkRdTqLlYt/JS0lmZ7X5Jj8uWQBAQGBePgGsPTXn8jLzWXwyCcBKCrTYK4y/Fd64YUXaN++PbNmzWLQoEFs3rz5hvlDNTFmzBi++uorBg0axIcffoiHhwcJCQn8888/vPbaa3h4eDBz5kymTJmCk5MTffv2JT8/n/379/P8888DVQZT+/btUalU2NrWnoz+sBPSuQcXDh8A4I8P3tTvz7x6mbiTx/jn85m4Nghm+NsfoVGrDYwhgNy0VFa8+wpDX5+JSUVouLSoEJ1OMK6VpqaUFRffcB2ZVy9j7+FltF+r0SCRSh+Y0unUuEusePtlfEKbY2nvSFr8JZIvRNc6/uSmdTUaRFqNhovHhAeGBm07ElOhsXN293bajXjiziz+HqIuKeH4hn8B4TOlRyIRXNkI4bLKhGln/0Ds3D3JSryCX/NWD8zn40GjTgbR4MGD6zSZRCIRn0QfAkJDQ5k9ezZffPEFb775Jp06deKzzz5j3LhxtzSvo6MjixYt4q233mLOnDk0b96cWbNm3dCIXrx4MUVFRXz22Wd89tln+v2dO3dm165dtZ537NgxA6/JrA/fBmDg8NF8+t1cZFIZCbEXmPG/cWRlZWJjY0dIaBi//bWRgKCGBnN99eUXfPvtt5yMiMDTx4/vFq7A1k7QELqUXoCzlQnOVlWNFNu0acP8+fN5//33ee+99+jRowfvvPNOnQzA6piZmbFnzx5ef/11hg4dSn5+Pu7u7nTv3l3vMRo/fjwlJSV88803vPLKKzg4ODB8+HD9HF9//TUvv/wy8+fPx93d3ahk/1HCJ7R5rcf++XwmAMkx54nas5PoA3tqHJccc54j//5JxycmcHrbf2QlXgXA1tWNp7+dR/LFaC5HnqJZ78e4cjaSf2d9jJOPP4279mDHb78AELFlA+qSUhq0bY9fWEvyMtKYP1UoOGg1aDgdn5hw+170HUCn05F4/iyrZr4BUGd9p8ToKMpKilGaGIaWr547S0l+HqaWVjz2/CsEtGzDxjlfcXb3dtoOH/3QGQCXrwkPAoz+6CuUpmas/uJDzK1t6P7Mc/pjEomEruOe4ei6v2ne7/YXBIgISHR3qwfCA0xeXh7W1tbk5uYahS1KSkqIi4vD19dXX3Uncn+SX6ImLqMQAB97c65mF+NkpcJSJdf3UqsJLzszbMyEROeycg2Xs4qRSMDeXMnlrKok80ZuVpRrdJjchjwukVuntv/N09s3sXXeD9c919bNg+ykq/rtl39fy4G/fufQ378Lx13d6DJ+Eqs//0A/pnm/QXQdf/1UgpjD+1k3+zODfS/9/i/7Vy3jyJo/9fueX/ynkdFwP3F6+2a2zrt+mLDyPTS3sWXQq+/w71cfU5iTTb9pM0hLiOPUlo2M/ngW2vJyovbs4MR/awnp3IM+z01HXVbK3ElPoi4pZtQHX1JSmI9Xk2b6sFx10hPisLCzx9Sy5pDy/ci2X38yaLvh7BfIE5/MEkvo7wDXu39fy4Ob6CIiUk8sVHIcLVXIpRKsTBU0qtY3rbGbNZeziihWa3CxMkEhl2KhMv73UMplBDgJoZJrnyViUgoo12oNDCiR+4+m3fvgH96ajd/PQiKV0mbYKFa9/7rBmOrGUOexE5FIpbQcOJTMKwlcOHKA7OQkA2MI6lYC7de8FSpzc0oLC/X7Ek6dJCn6nMG4mEP7adylx828vLvCjYyhyXMXIZXKOLZ+NWF9BmDl4EiT7n049PfvnNm1lctnBA/JklenGZznFy7IRSiUKvzDW3F+/25Wvi9IV4R07k6bYaP544M3ady1B+1GjCHlYgzL35mBa2AQT3w06w680tuPTqcjLkLInez29BTcGjSsMVFf5O5T5xKVfv36kZubq9/+/PPPycnJ0W9nZmbSqFGj27o4EZHbiUQiwdXaFEdLY0+eVCrBx8Gchq5W2JorazSGapov0NkSZYWgZHlFNV11r5HI/Ym5jS0j3v2E4W9/hEdwiH5/h1Hj8KuW49J1/CRa9BeUzpUmpgyc8RaNOnWrcU6Pho1veF25QoFPU8Ow3fGN/5IYHQWAX3PBIDi7axsABVmZ9135eUGWoSDp4NfexcbFFalMjkfDxrQcNBxLOwfMbWzp/OTTWDkIWlgNO3QG0BtDNeHdpKoKLahdJ4NjZ3dvZ9/KJeRnpnPwr98pKy4i5vB+0OlIjjnPwb9+Z8svc8jLSLt22jqTcSWBSxUJ4beT3csW8tOkMWQlJZJxOZ689DRkCgWNO/cQjaH7iDp7iDZv3kxpaal++9NPP9W3EwAoLy8nOrr2ZDoRkYcRU4UMX0dzolMMQ26lag0qMXT2wPDk598Re/wILQYMIfbkMX2VVGDr9kZjG3ftSdSeHQb7mvV+rM7ii+H9BxN9aB8SiQSdVkvC6ZMAQUWJxgAAgT9JREFU2Ll70uOZqcyf+jRXz53hvx9n668z5tNvcPEPvJWXeMuUFhWx7pvPSIuPBcAtqBGjPxQkBVwDgigrKcHGuXbxTzs3D5z9Amo18BQqE1RmVe17apImqJ7X9f2Exw2OHfhTkKaI3LEF14AgOo15Co9GNzZSK9Fptfzz+UzyM9J5/P3P8GzUpM7nXo9ytZpTWzaiLi0hYvN6FCoh7OcTGo5CTLO4r6izh+ja8ICYeiQiIqCSy5BgmPSZXSQ2Zn2QcPb1p+3w0cjkCvybt6Jp9z60HzkWS3sHo7EeDRvToE0HQPCOvLTiX7o9NaXO13INCOLJT7/hf3MX4+RT5R3wDWuBpb0D3k2bARgYXdXzTe4VUXu2k3D6JMV5QqQguH2VB8fM2ua6xlAlwe07639vMWAooT376vf1nDTVYKxMLqdx1543tdbki9Gs+uANtNq6F/mkxl0iP0No6rxj4c+kX46/qWtfy9WoSNSlJYBQZXfkX6GVUcMOXW7L/CK3DzGHSETkNuDnaE5ybgkWKhlp+aVkF6lxtjLRV8fkFaspKdfgaKF66CpmHjakMhk9J0+r9bhEImHAS2/c0jWc/QTdLKHy7BJQpToc0qWHvpN5JXERx9FqNXc96bYgK5PC3BzsPby4fOaUwbHAlvVv1RTcvjMH/lyBysyMDqPGIpMr0Ol0dBrzVI3GZ5dxk1CamuHdtJk+Z0sqk6PVVAmmyuRyLOwdyE1NMTp/6esvMuS197BydLrh2qqHyjKuJLDk1WlMXbBSL68AcG7vTjb+8DUmFpY89qKQ27Rt/g9YOTjRZfwklCamnN+/m6L8XBLPR9Fq0AjWf/t5jderzJcSuX+os0EkkUiMvsjFL3YREQFzlZwAJwu0Oh2ZFUrYBaXlWJoo0Gp1XM4qQqvTYSKXYWUq9rUSEQju0IWDf/2OXKnCPVjIwQxo0UZ/3NTKGnVpCYXZWVyOPHVd2YDbTWrsRZa9OR0Qquqyk5P0x1wDg7CokJ2oDxa2doz7Yg4yhQKZXPg/kEgkNRpDACozM33lXkjn7pzdvZ2+U1+ivKyMzT8LraI6jBpHcIculBYWknj+LFvnV1UQZlyOZ/60p7F1dSf8scH4hbfk+PrVhPcfgqWd4TVjTxjnDm38/iu6T3wWaycXSosK2VihQF5SkM/fn7yrH5eblsrS118wOr+6MWRhZ6/Pv/JtFl5jxZzIvaXOZfdSqZS+ffuiqoh/rlu3jm7duulbDZSWlrJp06aHUodILLsXqQ+JOcVkFpQil0rxsjdDq9URnylUFVmZKPBxML/BDCK3gwflf7MwJxuJVIqZlbV+34E/V3Do75WMePdjYg4fIGLzehq07ciA6a9fZ6bbR0FWJr88O77GYwNeegPXwOBajZg7hbqslKyrV/TetbLiIuJPnSCgZVukMkPPWcqlCyx/66Va53Ly9Wfs51W9N/OzMpj37ASQSOgwciz7Vi4xGN+4a0/O7Nx6S+t/dv5yjq1fjbqkmPaPjzXwPIncOe5I2f348Yb/HE8++aTRmFsV7hMReRiwNVOQWVBKuVZLbHoBViZVHqH8knLUGi0KmdiDTETA3MZYObzt8NG0GjQcuVKJytyCiM3riTm4l5g27Qls3b7e3vnctFQ2zPmS0J79aNihi5EBcS2b5n5b436vxqH6/Km7jUKp0htDAEpTs1rX4uwXQIPW7YUqtBpIi7tESUGB3iiJPS4IS7oGNKD1kMfxC29lIAlwq8bQoFffxczKmk73ueDmo06dDaLffvvtTq5DROSh4doGu3klQuNXmUSCRqcjLb8UCWBtqjBq+QFCwUJKbgkqhRQ7c9Gt/igikUiQKwUtKycfP3111rpvPqfv1JdrLf2/Fp1OR2lRIXt/X0zyhWiSL0Sze9lC+jw7Hd+wFjUaVurSEn3lG0Cf515i00/fANBzUu25VfcTEomEAS+/ydG1f7Nnec33rvP7d9Os92NAVbjMP7w1AI5ePoR07sHZ3duMzhv14VeY29iScSUBJx9frBycSIuPZduCn0iOOY9noyZY2DsQ2qMvOq22XpVuIvcWMala5I7TpUsXmjVrxrfffgsIvbWmT5/O9OnT7+m66kN8fDy+vr6cPHmSZs2aXXesRCLBw9aMq9lVekQyiQRnaxOScopZuPA3vvrgTfadTQBgwZwv2b1lIzv2H6FErcFcKSO9QJC4sDJRIL/Gm6TWaEnPL8XBQolSLpb2PwqEdO6uL1ffuXh+nQ2io2v/Zu+KRQb7ivNyWf3FB6jMzRny2vuozM2RSKXYuwsNl6snTw965R0CWrYhqF0nZHL5A5c32mLAUEI6d8fM2ob/fviaqL079cfO7NqKX3hLCnOyuRwpvOZKHSiAnpOn0qRbL/79+hN9Zd3I9z/HvaKtT/WqOicfP0Z/8CXpl+Nx9PJBIhU9wA8i4l9N5K5z9OhRg471EomENWvW3LsFUVU0cO3PV199dVPz2ZkrCXK21N9ALE0U2JjVnEw9euJzrF6/iavZRWQUlJJQTdgxu0htND4tr4SMglKSckrqtaa6vs8TJkyoc/9CkbtDo07d9ZVSJQX5dS4Jv9YYqk5pYSEr33+NFe+8woq3X6a4QNDSqqy2ata7PwEthQRvuULxwBlDIHzmzaxtAOj+zHO0HzmWMZ/MRiqTkRp7kflTn2bF2zMoV5dh6eCIg5eP/lyZXIF7cCOenbeMoHad8G/RGrdrehwaXEsqxcnHTzSGHmDEv5zIXcfR0RGzagJsdaGs7M7q+iQnJxv8LFy4EIlEwrBhw256TpVChm1FRZmNuQK5VIpdDS09zMwtKJHV/H5kF5UZCKLqdDrySoSS4/wSNepy7U2v71ZRq42NNZE7g8rMjEk/LCSwVTsAIndsvuE5OSnJBtuugUG0GTbKaJy6pJiy4mLO7dmBTqvVi1L6t2h9G1Z+/6A0MaXN0JG4BDTQh8aq4x9ecxd5iURC/xdfY/Cr794w90rkwUY0iB5hunTpwvPPP8/06dOxtbXF2dmZ+fPnU1hYyFNPPYWlpSUBAQH8999/+nPOnDlD3759sbCwwNnZmbFjx5KRkaE/XlhYyLhx47CwsMDV1ZWvv/7a6Lo+Pj4G4TOAIUOGIJFI9NszZ86kWbNm/PrrrwZVQpcvX2bQoEFYWFhgZWXF448/Tmpqqn7uyvOWLl2Kj48P1tbWjBo1ivz82pu3Ari4uBj8/Pvvv3Tt2hU/Pz+DcefPn6ddu3aYmJjQuHFjdu/efd153WxNObz5Hxo38MfMzIxpT49Bri5EKpGgqgh3zZ39OUN6VCkiv/vSc0yfOIb5c2bRoWkDgoKCAbhy5QpDho2gdbAnHRv78sLTT7Dr+FkKS6s0WRYuXEhISAgqlQpXV1emTZt23ff5WmbOnMnixYv5999/9V6yXbt2ER8fj0QiYdWqVXTu3BkTExOWL1+uf7+r8+233xrMX+lxmjVrFq6urtjb2zN16lQDg6q0tJTXX38dT09PVCoVAQEBLFiw4Lrv7aNIk269ADi3ZyflN3hIqOyXBYBEQsfR42n/+JNM+vE3fe5MdU5v30z0oX0U5mSjNDXF8yHOfal8H6vj17xVDSNFHiVEg+hOoNNBWeG9+amngvjixYtxcHDgyJEjPP/88zz77LOMGDGCdu3aceLECXr16sXYsWMpKioiJyeHbt26ERYWxrFjx9i0aROpqak8/niVhP6rr77K7t27+ffff9myZQu7du3ixIkTtV7/6FGhuuO3334jOTlZvw1w8eJF/v77b/755x8iIiLQarUMGjSIrKwsdu/ezdatW4mNjWXkyJEGc166dIk1a9awfv161q9fz+7du/n885rF0WoiNTWVDRs2MHHiRKNjr776KjNmzODkyZO0bduWAQMGkJmZWcMsFa/vyBGmTJ7EtGnTiIiIoFu3rnz5+acABLlY0tTDBrm06qnU1kyJiULG0QN7SEqI5ecV/7BwxV+o1Wp69uqFRGnKb39tZOmazZiZmzN5zDDOJ2ah1emYO3cuU6dOZfLkyURGRrJ27VoCAgJu+D5X55VXXuHxxx+nT58+em9Zu3bt9MffeOMNXnzxRc6dO0fv3r3r/J7u3LmTS5cusXPnThYvXsyiRYtYtGiR/vi4ceP4/fffmTNnDufOneOXX37BQixLNsI7NAxLe0dKCgu4cOTAdcfGRRwHoP3Isfzvp0V4hjQFwMrBke5PP8v4WT8ajM+8epkN3wmtOHyatdDrBD2MeIeGIVdUeWttnF1vW6sOkQcXMan6TqAugk/d7s2130oCZd11bkJDQ3nnnXcAePPNN/n8889xcHBg0iRBDO29995j7ty5nD59mm3bthEWFsann36qP3/hwoV4enoSExODm5sbCxYsYNmyZXTv3h0QDC4PD49ar+/oKDR+tLGxwcXFUPq/rKyMJUuW6Mds3bqVyMhI4uLi8PQUEkCXLFlCSEgIR48epWVLISFSq9WyaNEiLC0tARg7dizbt2/nk08+qdN7snjxYiwtLRk6dKjRsWnTpunDaHPnzmXTpk0sWLCA1157rca5vvvuO/r06aM/3qBBAw4cOMCmTZv0Y0yVgqdIKpHgbmOKqVKGubk58+bPJzFPjUwq4fffV6Iu1zDzqzlIJBJcrE1w//rH/7d373E93X8Ax1/fLt/u99JNCtGFJOUSJibCmJm5rbnNZcytXdh97hMbNmazH4aZzZjrzP06cilRbhEpZUroXnQ9vz++6/BVUUTR5/l4fB8653zO53zO+fr2ffe50raRE2FHD9PAtgczZszggw8+YMKECXLexc/kYc/5foaGhujp6ZGbm1tquqCgoFKfy6OYmZnx/fffo6mpiaurK6+88gp79+5lxIgRREdHs3btWnbv3o2/v2qF9wdr5gQVDQ1NGnfw5+ifv3Nm364yl3/4e8HXxJ5S1RDV92lZ6iSKlg6ONO/5BjlpqUiSpLZUSGnriL1INDQ0eSv4W24lxFPPuzkK7o3qE2ouUUNUwzVp0kT+WVNTEwsLCzw87v2lZG1tDUBycjKRkZHs378fQ0ND+eXqqmrOiYmJISYmhry8PFq2vNc+b25ujouLy2OVzdHRUf4iB4iKisLBwUEOhgDc3d0xNTUlKipK3ufk5CQHQwC2trYkJ6tWwF69erVa+Q8dOlTiuj///DOBgYGlTubn63tvuQItLS18fHzkazdq1EjOt2vXrnKZ738eD+YBoKutiYZCgbWxLhr/1RZ5eHhgbqSPUlODwiKJsJOnuBp7BV9XB1q51KaerSXtmtQjN/cu167GEh13jevXr8uBaHnEx8erPYv7A92y+Pj4lDv/+zVq1AjN+/pf3P+eREREoKmpiZ+fX1mnC/dp3KETKBQknDtNatL1EsdzMtK5EHKvKdfSwbHMvNq9OYQu775Hs2490dBU/X2sa2gk91V6kVnUroOLb1u0lToiGBIAUUP0dGjrq2pqquraFUn+wArdCoVCbV9xJ8OioiKysrLo0aMHs2fPLpGPra0tly+Xvor14yqeBb2iSrunoiJV5+NXX31VLUCxt7dXS3vo0CEuXrzIH3/8UeHrbtu2Te4Xo6enV+7zNBQKlFoaWBndm3PIwMAAhUKBqYGS5Iy73ExNx82jKbMXLsG5lqEcOOXmF5Im6VKgqHhnTzs7OyIiIuRtc3PzR57z4HuioaFRYqHn0jpbP+w9qcizEsDYshZOns2Iiwjn3IE9tO2vPiHu/eugNe/5RrlGh1nXrc+wBUvQ1tVFz9DokekF4UUkAqKnQaGoULPV86JZs2asX78eJycntLRK/tepX78+2traHD9+nDp16gCQmppKdHT0Q//619bWLteSL25ubiQkJJCQkCDXEp0/f560tDTc3d3LdQ9GRkZqtUcPWrZsGd7e3nh6epZ6/NixY7Rrp1rlu6CggPDwcLnjsqNjyb/E3dzcOH78eIk8ystcX5vkjLu4NGrC9s0bqG1vQ8O6tmppLidnkZNXQB1HR/bu3UuHDh1KzevB56ylpSX3MbqfUqks9xI8VlZWJCUlIUmS/MV7f5BVHh4eHhQVFXHw4EG5yUx4uEZ+HYmLCOfCkX9o02+g/OxzMtLZ9d8aX65t/Gjbb2C58zS2tHp0IkF4gYkmM6HcxowZQ0pKCgMGDCAsLIyYmBh27tzJ0KFDKSwsxNDQkGHDhjFx4kT27dvH2bNnGTJkCBqPmJfDycmJvXv3kpSURGpqapnp/P398fDwIDAwkJMnTxIaGsqgQYPw8/N77Kac+2VkZLBu3TqGDx9eZppFixaxceNGLly4wJgxY0hNTeXtt98uM/348ePZsWMH33zzDZcuXeL7779X6z/0KEotTQx1tOjWqw+m5ha8O7g/hw4dIjY2lgMHDjB+/HjupKpG2b37/sfMnTuXBQsWcOnSJU6ePMnChQvlvMr7nJ2cnDh9+jQXL17k1q1bDx1e3759e27evMmcOXOIiYlh0aJFaqMSy8PJyYnBgwfz9ttvs2nTJvne1q5dW6F8apL6zVqgpaND+o0kkmKi5f07fphPYYFq1KFn525imLggVEC1D4gyMzMJCgrC0dERPT09WrduXWKETFRUFK+++iomJiYYGBjQvHlz4uPj5eN3795lzJgxWFhYYGhoSO/evdWGagvlY2dnR0hICIWFhXTu3BkPDw+CgoIwNTWVg56vv/6al156iR49euDv70/btm3x9n54B825c+eye/duHBwc8PLyKjOdQqFg8+bNmJmZ0a5dO/z9/alXr95jNW+VZs2aNUiSxIABA8pMExwcTHBwMJ6enhw+fJgtW7ZgaVn2IpetWrViyZIlfPfdd3h6erJr1y65E3t5mekr0dPTZ/mff+Pk6Mjrr7+Om5sbw4YN4+7du9S2tkBDoaDr6/2Z881cfvjhBxo1akT37t25dOmSnE95n/OIESNwcXHBx8cHKysrQkJKXw8KVDVgP/zwA4sWLcLT05PQ0FA+/PDDCt0fqDqov/HGG7z77ru4uroyYsQIsrOzK5xPTaGtq4uzj2rSxHMHVZ2hczLS5Y7UAHYNXKukbILwvCr3avdVpV+/fpw9e5Yff/wROzs7fv31V+bPn8/58+ext7cnJiaGFi1aMGzYMAYMGICxsTHnzp2jVatW1Kqlmtl19OjR/P3336xYsQITExPGjh2LhobGQ3/R30+sdi9UpaIiidhb2WhralDHovQ+YvEpOaTl5GGip01BoYSxnjaWhkqupd5BqaWBtXHN+7/5on82r56O4M+ZquC6tltjGrZqw77lPwGqFemrahFWQahOKrLafbUOiO7cuYORkRGbN2/mlVfuTSTm7e1N165dmTFjBv3790dbW5tVq1aVmkd6ejpWVlb89ttvvPHGG4Bqcj03NzeOHj1Kq1atHlkOERAJ1V3W3Xyu3FKvUXE015eXAXGxMZIngqwpXvTPplRUxP/GDCUrRX0erBav9eGlAYOrqFSCUL1UJCCq1k1mBQUFFBYWlvhlpqenx+HDhykqKuLvv/+mYcOGBAQEUKtWLVq2bKm2XlN4eDj5+flqnTVdXV2pU6cOR48eLfW6ubm5ZGRkqL0EoToz0NFC+cAisPeviXYxKZOMO2KpjReJQkOj1Fqgul5P3p9OEGqiah0QGRkZ4evry/Tp07l+/TqFhYX8+uuvHD16lMTERJKTk8nKyiI4OJguXbqwa9cuevXqxeuvvy4vqZCUlIRSqcTU1FQtb2tra5KSkkq97qxZszAxMZFf9897IwjVkUKhwKSMxWOLxd3OLjFEXni+te7zptqaYzoGBqLvkCA8pmodEAGsWrUKSZKwt7dHR0eHBQsWMGDAADQ0NOR5THr27Ml7771H06ZN+fjjj+nevTuLFy9+7Gt+8sknpKeny6+EhITKuh1BeGosDHTQ0tBAT7vsprHihWGFF4OOvgGvTfyCxh1Ua3M5NmkmRpYJwmOq9vMQ1a9fn4MHD5KdnU1GRga2trb069ePevXqYWlpiZaWVok5aNzc3Dh8+DCgWrQzLy+PtLQ0tVqiGzdulLmEgY6ODjo6OqUeE4TqSqmlgauNEQoFpN3JJye3ACsjHbJyC7mWqmo+S81WdbwWXiwvvTkYPSMjPDt1q+qiCMJzq9rXEBUzMDDA1taW1NRUdu7cSc+ePVEqlTRv3pyLFy+qpY2OjpYnyfP29kZbW5u9e/fKxy9evEh8fHyJJRQE4XmnoaFaod5MX4m9mT5KLU3MDZQ0tFZNRplxN5/E9DsUFYmmsxeJvrEJ7QKHYlLLuqqLIgjPrWpfQ7Rz504kScLFxYXLly8zceJEXF1dGTp0KKBafbxfv360a9eODh06sGPHDv766y8OHDgAgImJCcOGDeP999/H3NwcY2Njxo0bh6+vb7lGmAnCi0BXWxMDpRbZeQXczMylsEiitlnFlnkRBEF4kVX7gCg9PZ1PPvmEa9euYW5uTu/evZk5c6a8NlKvXr1YvHgxs2bNYvz48bi4uLB+/Xratr03+mL+/PloaGjQu3dvcnNzCQgI4IcffqiqWxKEKmFuoCQ7T9WHKCU7D3tTvXKtcyUIglATVOt5iKoLMQ+R8CIoKpK4lJxFboFqnTInCwOMX+D+ROKzKQjCCzMPkfBiaN++PUFBQfK2k5MT3377bZWV52k5cOAACoWCtLS0J85rypQpNG3aVN4eMmQIr7322hPlqaGhwMXGCEtD1YCB1Jy8J8pPEAThRSICIuGZCwsLY+TIkfK2QqFQm0yzKmzYsIHOnTtjYWGBQqEodcX2d955h/r166Onp4eVlRU9e/bkwoULz6R83333HStWrKiUvMwNlABk3Ckgv7BI3h8XF1fmvT/owSBXEATheScCIuGZs7KyQl+/Yh168/Kebm1GdnY2bdu2Zfbs2WWm8fb2Zvny5URFRcmd/Tt37kxhYeFTLRuoBgc8OLno/SryfHS1NdFXaiEhkZbz9GavliSJggIx75EgCM8HERDVYO3bt2fcuHEEBQVhZmaGtbU1S5YsITs7m6FDh2JkZISzszPbt2+Xzzl79ixdu3bF0NAQa2trBg4cyK1bt+Tj2dnZDBo0CENDQ2xtbZk7d26J697fZObk5ASoOscrFAp5u7jJaOnSpWp9QOLj4+nZsyeGhoYYGxvTt29fbty4IeddfN6qVatwcnLCxMSE/v37k5mZ+dBnMXDgQL788ku1JV4eNHLkSNq1a4eTkxPNmjVjxowZJCQkEBcXp5YuJCSEJk2aoKurS6tWrTh79uxDrw0QHByMtbU1RkZG8ir293uwyax9+/aMHTuWoKAgLC0tCQgIAB79/hQVFTFnzhw6+3riU98aT1dnZsyYQUFhEXXr1gXAy8sLhUJB+/btSy3rkCFDOHjwIN999x0KhWqYf1xcnNxkuH37dry9vdHR0eHw4cOlNvcFBQWp5d++fXvGjx/PpEmTMDc3x8bGhilTpqidk5aWxjvvvIO1tTW6uro0btyYrVu3PvLZCoIglIcIiJ4CSZLIyc+pkldF+8ivXLkSS0tLQkNDGTduHKNHj6ZPnz60bt2akydP0rlzZwYOHEhOTg5paWm8/PLLeHl5ceLECXbs2MGNGzfo27evnN/EiRM5ePAgmzdvZteuXRw4cICTJ0+Wef2wsDAAli9fTmJiorwNcPnyZdavX8+GDRuIiIigqKiInj17kpKSwsGDB9m9ezdXrlyhX79+annGxMSwadMmtm7dytatWzl48CDBwcEVei6Pkp2dzfLly6lbt26JpV0mTpzI3LlzCQsLw8rKih49epCfX3ZNzNq1a5kyZQpfffUVJ06cwNbWtlyjIFeuXIlSqSQkJITFixeX6/355JNPCA4O5ovPP2fzvuN8tfB/aBmYEZWYwb5DRwDYs2cPiYmJbNiwodTrfvfdd/j6+jJixAgSExNJTExUewYff/wxwcHBREVF0aRJk0fex/33Y2BgwPHjx5kzZw7Tpk1j9+7dgCqQ69q1KyEhIfz666+cP3+e4OBgNMWszIIgVJJqP+z+eXSn4A4tf2v56IRPwfE3j6OvXf7mKE9PTz7//HPg3pelpaUlI0aMAODLL7/kxx9/5PTp0+zZswcvLy+++uor+fyff/4ZBwcHoqOjsbOzY9myZfz666907NgRUH3J1a5du8zrW1lZAWBqalpi5vC8vDx++eUXOc3u3bs5c+YMsbGx8hfwL7/8QqNGjQgLC6N58+aA6stzxYoVGBmpJiMcOHAge/fuZebMmeV+LmX54YcfmDRpEtnZ2bi4uLB7926USqVamsmTJ9OpUye1+9+4caNaYHK/b7/9lmHDhjFs2DAAZsyYwZ49e0rUEj2oQYMGzJkzR96eMWPGQ98fW1tbvvvuO77//nuGDh1KQkoO9o5O0MIXCdDUU43AsLCwKHMWd1A13ymVSvT19UtNN23aNPn+K6JJkyZMnjxZvrfvv/+evXv30qlTJ/bs2UNoaChRUVE0bNgQgHr16lX4GoIgCGURNUQ13P1/wWtqamJhYYGHh4e8z9paNfNtcnIykZGR7N+/H0NDQ/nl6qpaSDImJoaYmBjy8vJo2fJeMGhubo6Li8tjlc3R0VEOhgCioqJwcHBQq41wd3fH1NSUqKgoeZ+Tk5McDAHY2tqSnJwMwOrVq9XKf+jQoQqVKTAwkFOnTnHw4EEaNmxI3759SwQu98+AXnz/xeW7/9qjRo2S7+v+Z/ZgHmXx9vZW237U+xMVFUVubq4crJoZqAdy2Xkl+0IdOnRILb/Vq1c/slw+Po+32vqDtUn3v28RERHUrl1bDoYEQRAqm6ghegr0tPQ4/ubxKrt2RRRPcFlMoVCo7SueuK+oqIisrCx69OhRasdjW1tbLl++/BglLpuBgcFjnVfaPRUvBPzqq6+qBR/29vYVytvExAQTExMaNGhAq1atMDMzY+PGjQwYMKBc598/gutRc2I8yoPP51Hvz5UrV9TPV2qi1NIgr6CoRPpiPj4+amUuDpArUi4NDY0STbmlNSE+7H3T06vY/2tBEISKEgHRU6BQKCrUbPW8aNasGevXr8fJyQktrZL/derXr4+2tjbHjx+nTp06AKSmphIdHY2fn1+Z+Wpra5drpJabmxsJCQkkJCTItUTnz58nLS2txAK/ZTEyMlKrPXoSkiQhSRK5ublq+48dO1bi/t3c3ABwdnYukY+bmxvHjx9n0KBBanlU1KPenwYNGqCnp8fevXsZPnw4CoWCWka6XEvNwUxfSfJ/AUn+fSPD9PT0Si2zUqks9+g6KyurEh3LIyIiSgRAD9OkSROuXbtGdHS0qCUSBOGpEE1mQrmNGTOGlJQUBgwYQFhYGDExMezcuZOhQ4dSWFiIoaEhw4YNY+LEiezbt4+zZ88yZMgQNDQe/t/MycmJvXv3kpSURGpqapnp/P398fDwIDAwkJMnTxIaGsqgQYPw8/N77GaaYikpKURERHD+/HlAtQBwREQESUlJAFy5coVZs2YRHh5OfHw8R44coU+fPujp6dGtm/oK49OmTWPv3r3y/VtaWj50UsUJEybw888/s3z5cqKjo5k8eTLnzp2r8D086v3R1dXlo48+YtKkSfzyyy/ExMQQfeYkoTv+pLaZ3n+jt/TYvHUbN27cID09vcxrOTk5cfz4ceLi4rh165Zck1Oal19+mRMnTvDLL79w6dIlJk+eXK6Rd/fz8/OjXbt29O7dm927dxMbG8v27dvZsWNHhfIRBEEoiwiIhHKzs7MjJCSEwsJCOnfujIeHB0FBQZiamspBz9dff81LL71Ejx498Pf3p23btiX6ujxo7ty57N69GwcHB7y8vMpMp1Ao2Lx5M2ZmZrRr1w5/f3/q1avHH3/88cT3tmXLFry8vHjllVcA6N+/P15eXixevBgAXV1dDh06RLdu3XB2dqZfv34YGRlx5MgRatWqpZZXcHAwEyZMwNvbm6SkJP76668SHa/v169fP7744gsmTZqEt7c3V69eZfTo0RW+h/K8P1988QUffPABX375JW5ubvTr14+bN2+iUCiwMtbno2nB/Lx0CXZ2dvTs2bPMa3344Ydoamri7u6OlZUV8fHxZaYNCAiQ76958+ZkZmaq1YaV1/r162nevDkDBgzA3d2dSZMmPZM5oARBqBnEWmblINYyE2qCvIJCLiSp5mtysTFCR+v5HtIuPpuCIIi1zARBqDClliZGuqp+PZeTs8i48/RmsRYEQahuREAkCILM3EAVEBUWScTdzqagsOy+QYIgCC8SERAJgiArriEqli5qiQRBqCFEQCQIgkxDocDO5N6cPynZT3dRXUEQhOpCBESCIKixNNLBzcYYhULBnfxC7uSJFesFQXjxiYBIEIQStLU0MNZVTe6Yki2azQRBePGJgEgQhFKZ/7fWWdqdPIqKxOwcgiC82ERAJAhCqQx1tFBqaVBYJJEmOlcLgvCCEwGRIAilUigUmOuraomupeZwPe3OQ9MXSRIJKTnczMx9aDpBEITqSAREwlPXvn17goKC5G0nJye+/fbbKivP03LgwAEUCgVpaWlVXZRKY2Zwb8mRW1m55JTSwfpufiGZd/PJultAak4eSel3yCsQ8xcJgvB8EQGR8MyFhYUxcuRIeVuhULBp06aqKxCwYcMGOnfujIWFBQqFgoiIiBJp3nnnHerXr4+enh5WVlb07NmTCxcuPPvCPkPamhpqS3ikZKkPw5ckiSs3s5n06Re08Gmm2gek5pQ+XL+gqIi0nDyKxIpBgiBUMyIgEp45Kysr9PX1K3ROXt7TnQ8nOzubtm3bMnv27DLTeHt7s3z5cqKioti5cyeSJNG5c+cXeoHRvLw86lrqo6utCorS7uSrzV59KyuPglJWur+RcZfYW9klOmMnpd8lPiWH5DKa1Z72+ywIglAWERDVYO3bt2fcuHEEBQVhZmaGtbU1S5YsITs7m6FDh2JkZISzszPbt2+Xzzl79ixdu3bF0NAQa2trBg4cyK1bt+Tj2dnZDBo0CENDQ2xtbZk7d26J697fZObk5ARAr169UCgU8vaUKVNo2rQpS5cuVVucMz4+np49e2JoaIixsTF9+/blxo0bct7F561atQonJydMTEzo378/mZmZD30WAwcO5Msvv8Tf37/MNCNHjqRdu3Y4OTnRrFkzZsyYQUJCAnFxcWrpQkJCaNKkCbq6urRq1YqzZ88+9NoAM2bMoFatWhgZGTF8+HA+/vhjmjZtqpZm6dKluLm5oauri6urKz/88IN8LC4uDoVCwYYNG+jQoQP6+vp4enpy9OhRtTwOHz7MSy+9hJ6eHg4ODowfP57s7Gz5uJOTE9OnT2fQoEEYGxszcuRIlFqaLJ07nZ5+PjR3tqW+c30++uQzwmNvkph+h81rf2Px/NlcPH8WTwczPB3M2Lz2NzLv5nPm4mW192vYoEBu30wmJVtVS1TW+ywIgvCsiYDoKZAkiaKcnCp5SRVsili5ciWWlpaEhoYybtw4Ro8eTZ8+fWjdujUnT56kc+fODBw4kJycHNLS0nj55Zfx8vLixIkT7Nixgxs3btC3b185v4kTJ3Lw4EE2b97Mrl27OHDgACdPnizz+mFhYQAsX76cxMREeRvg8uXLrF+/ng0bNhAREUFRURE9e/YkJSWFgwcPsnv3bq5cuUK/fv3U8oyJiWHTpk1s3bqVrVu3cvDgQYKDgyv0XB4lOzub5cuXU7duXRwcHNSOTZw4kblz5xIWFoaVlRU9evQgP7/sUVqrV69m5syZzJ49m/DwcOrUqcOPP/5YIs2XX37JzJkziYqK4quvvuKLL75g5cqVauk+++wzPvzwQyIiImjYsCEDBgygoEDV7ycmJoYuXbrQu3dvTp8+zR9//MHhw4cZO3asWh7ffPMNnp6enDp1ii+++AIAY2NjfvhpKRv2HePjKcEsW7aUX5eqArKAHr0YNHIsDVzcuHL1GqHnYgjo0YuioiIG9H2DlJQU9h84wA+rN3DtahyT3n2bgsIiMv8bufbg+ywIglAlJOGR0tPTJUBKT08vcezOnTvS+fPnpTt37sj7CrOzpfMurlXyKszOLvd9+fn5SW3btpW3CwoKJAMDA2ngwIHyvsTERAmQjh49Kk2fPl3q3LmzWh4JCQkSIF28eFHKzMyUlEqltHbtWvn47du3JT09PWnChAnyPkdHR2n+/PnyNiBt3LhRLd/JkydL2traUnJysrxv165dkqamphQfHy/vO3funARIoaGh8nn6+vpSRkaGnGbixIlSy5Yty/VMYmNjJUA6depUqccXLVokGRgYSIDk4uIiXb58WT62f/9+CZDWrFlT4v7/+OOPMq/ZsmVLacyYMWr72rRpI3l6esrb9evXl3777Te1NNOnT5d8fX3Vyr106VL5ePGziYqKkiRJkoYNGyaNHDlSLY9Dhw5JGhoa8v9fR0dH6bXXXiu1nAWFRdKZa2lSZEKq9P5n0yR3j6ZSZEKqFJmQKn3xxZdyefMLCqWY5Exp8eoNkqampnTx8hUp806eFJmQKm3Ye1QCpNV/7ZVikjNLfZ8rS2mfTUEQapaHfX8/SNQQ1XBNmjSRf9bU1MTCwgIPDw95n7W1NQDJyclERkayf/9+DA0N5Zerqyugqn2IiYkhLy+Pli1byuebm5vj4uLyWGVzdHTEyspK3o6KisLBwUGtRsbd3R1TU1OioqLkfU5OThgZGcnbtra2JCcnA6qalvvLf+jQoQqVKTAwkFOnTnHw4EEaNmxI3759uXv3rloaX19f+efi+y8u3/3XHjVqFAAXL16kRYsWanncv52dnU1MTAzDhg1TO3/GjBnExMSonXf/+2lrawsg33tkZCQrVqxQyyMgIICioiJiY2Pl83x8fErc9x9//EG7l9rycrOGtHKpzfffzCTx+jUcLfRpUtsUDQ2FnFZLU4N6VoYkXY3B2s4efXNrMnNVtVT1G7piYmpK7OVosnILKCgsKvE+C4IgVAWtqi7Ai0ihp4fLyfAqu3ZFaGurr26uUCjU9ikUqi+6oqIisrKy6NGjR6kdj21tbbl8+fJjlLhsBgYGj3VeafdU9F/H31dffVUtYLO3t69Q3iYmJpiYmNCgQQNatWqFmZkZGzduZMCAAeU6//4mIWNj43Kdk5WVBcCSJUvUyg6qIPZ+Zb13xfm88847jB8/vsQ16tSpI//84HM/evQogYGBTJ06Fb+O/qTma7Fj8wZWL12EiZ7ywaxk+jqqst2/QGwdc30UIHfSvpNf+NjvsyAIQmUSAdFToFAoUFRwFNXzoFmzZqxfvx4nJye0tEr+16lfvz7a2tocP35c/oJNTU0lOjoaPz+/MvPV1tYu10gtNzc3EhISSEhIkGuJzp8/T1paGu7u7uW6ByMjI7XaoychSRKSJJGbqz5i6tixYyXu383NDQBnZ+cS+bi4uBAWFsagQYPkfff3pbK2tsbOzo4rV64QGBj42OVt1qwZ58+fL7UMD3PkyBEcHR357LPPAEhMu8PqnxaqpVEqlSXeQ8/Gjbhx/V+Srl/Dxq42APEx0aSlpeHtqaqFvJsv5isSBKF6EE1mQrmNGTOGlJQUBgwYQFhYGDExMezcuZOhQ4dSWFiIoaEhw4YNY+LEiezbt4+zZ88yZMgQNDQe/t/MycmJvXv3kpSURGpqapnp/P398fDwIDAwkJMnTxIaGsqgQYPw8/MrtZmnIlJSUoiIiOD8+fOAqhkrIiKCpKQkAK5cucKsWbMIDw8nPj6eI0eO0KdPH/T09OjWrZtaXtOmTWPv3r3y/VtaWvLaa6+Vee1x48axbNkyVq5cyaVLl5gxYwanT5+Wa3gApk6dyqxZs1iwYAHR0dGcOXOG5cuXM2/evHLf40cffcSRI0cYO3YsERERXLp0ic2bN5foVP2gBg0aEB8fz5o1a4iJiWHdL0vYvf0vtTROTk7ExsYSERHBrVu3yM3NpVOnTri5N+aTcSOJOhPJ5bMRvD10CH5+fvi1aYm2pgZFkkShWCdNEIRqQAREQrnZ2dkREhJCYWEhnTt3xsPDg6CgIExNTeWg5+uvv+all16iR48e+Pv707ZtW7y9vR+a79y5c9m9ezcODg54eXmVmU6hULB582bMzMxo164d/v7+1KtXjz/++OOJ723Lli14eXnxyiuvANC/f3+8vLxYvHgxALq6uhw6dIhu3brh7OxMv379MDIy4siRI9SqVUstr+DgYCZMmIC3tzdJSUn89ddfKJVlNy0FBgbyySef8OGHH9KsWTNiY2MZMmSI2hD04cOHs3TpUpYvX46Hhwd+fn6sWLGCunXrlvsemzRpwsGDB4mOjuall17Cy8uLL7/8Ejs7u4ee9+qrr/Lee+8xduxYmjZtypEjR+TRZ8V69+5Nly5d6NChA1ZWVvz+++8oFAq2bNmMubk5b7/xCkP6viq/XwqFQl48VgREgiBUBwpJElPGPkpGRgYmJiakp6eX6Pdx9+5dYmNjxRwqQqXq1KkTNjY2rFq1qqqL8tTkFRRxMSkDCWhobST3K6osT+uzmZCSQ1pOPh61TSotT0EQno6HfX8/SPQhEoQqlpOTw+LFiwkICEBTU5Pff/+dPXv2sHv37qou2lOl1NLASFebjLv5pGTnYWdasQEBz1pi+h3eXX2SU/FpAPw9vi2N7ERQJAgvCtFkJghVTKFQsG3bNtq1a4e3tzd//fUX69evf+is2S+K4maz1Jy8Est8VBe5BYXsOJvEO6vC5WAIYNnh2LJPEgThuSNqiAShiunp6bFnz56qLkaVMNLVQqmpQV5hEel38jEzKNnXqkiSyMktwEBHS+5onlugGtF2/8KzFRWRkEY9KwN0tDSYtzsad1tjejYtOQ3Djwdi+HbPpRL7t51J5Mvu7pjql90/TBCE54cIiARBqDLFnauTMu5yOzuv1IDoVmYuSRmqyS8tDXUwN1By5WYWkgQuNkZoaZavovtGxl2+3XOJVzxsGbYyjNyCIlrXt+BIzG05zUsNrDA3UCJJEssOx+JuZ1wiGJoY4MLvofFcS73DuhPXGNGu3hM8gYeTJImD0TfxdjTDSFf70ScIgvDYREAkCEKVMjNQciMjl5y8Au7kF6J3X+dqSZLkYAjgVlYut7LuzfuUmpOHlVH5OkxP3nyOHeeS+D00Xt53fzAEsD5cFeAcjL7JjL+jMNJR/xXp19CKMR2csTBQ8vGGM6w6dpVhbeuqzdRdWTLv5uMxZZe83btZbb7p00RtOgZBECqP6EMkCEKV0tbUwFhPFXikZOepLVB8+WbWQ89NysglPUd1zt38QvnfgiL1CR8lSWJP1I1HluW30HgkSeKf6FsA8pIjAC3rmjO5h2oC0J5N7THW1SI+JYeD0TfLd6MV9MvRq2rb609eIyyu7Hm6BEF4MiIgEgShyhV3rr6dlcuZf9NJSr9Dbn4hd/IePoO5JElcTckh5mY20TcyiU/J4dKNTOJu5agFVnG3cyh4oNN2Z3dr+ee5fTwx1NEi9lY2Lb7ay88h6h2mh7Wtyx/v+FLPyhAAPaUm/ZqrZktfeTTuse/7fsmZd/l04xlm/n2e+Ns5HLtyu0SatScSKuVagiCUJAIiQRCqnOEDTVPJmblEJ9+rHbI308PD3gQnCwOMdbXlV7GcPFVNTvqdfKT/tuNu5ZBfWMStrFw6fHMAgEZ2xnzVy4ML07uw8E0v6pjrY2mopFMja17zUk1QeTNT1SSnUICOlupXZNsGliXK/FYrRxQKOHDxJhtOXuPlbw6w78Kja6HKMvjnMH47Hs+SQ7G0+3o/hy6paqlGt68vp/n7dCLpd/If+xqCIJRNBESCIFQ5hUKBtbF6X6DiGp5aRrpYGOigUCgw1tPGydIAJ0sDbEwe3ncor7CQ5Ixc1p+8Ju8LaGTDmy3roKutiY6WJn+Na8ue9/0w1tXmzRaO6mUCvuvflPEdG+DXwKpE/o4WBrRvqNr/4bpIrtzKZubfUZRnrttDl24y7a/znLuezsfrTxMWl0JUYkaJdDbGukwKcCF2VjdcrI24k1/In+HXSslREIQnJQKiGqx9+/YEBQUBqrWovv322yotT1VQKBRs2rTpifM5cOAACoWCtLQ0AFasWIGpqekT51uT1DLSwd5UD40HOg0b6ZY+9kNXWxNXG+OHjr6SgD3n79XavOqpvkyJiZ62PGze3c6YN1vWkY+NfbkBXRrb8n6nhmV2mh7U2gmA4ta4mJvZHL58q8zygCrQG7gslJ9DYnllwWHWhCXQZ/HRUtO62RqpFotWKBjUWhWwrToaV23nbBKE51m1D4gyMzMJCgrC0dERPT09WrdurbYS+P1GjRqFQqEo8cWekpJCYGAgxsbGmJqaMmzYMLKyHt5ZU6g+Dhw4QM+ePbG1tcXAwICmTZuyevXqR57n5OQkf5nc/xozZsxTL3O/fv2Ijo5+6td5kQJZhUKBhaEO7rbG2PxXW6SpUKCnLHuuIaWWBrYmuugpNbE31UNXSxNTPSWmeveG76dk5wGwdVxbnCwNHlqGr3p5cG5qAPP6evLufU1VZfFrYIWjhb7avpVH4h56Tuyt7DKPjXvZmbXv+PK6lz0O5npM6uIqH3utqT1GulrE3c7hn0tPpyO3INRk1X7Y/fDhwzl79iyrVq3Czs6OX3/9FX9/f86fP4+9/b1J1DZu3MixY8dKXagyMDCQxMREdu/eTX5+PkOHDmXkyJH89ttvz/JWhMd05MgRmjRpwkcffYS1tTVbt25l0KBBmJiY0L179zLPCwsLo7DwXqfcs2fP0qlTJ/r06fPUy6ynp4eeXtlLUeTl5T10wdeaTENDFRjdyS/EUEerRI3Rg3S1NWlQywgAC0Mdeb+hdhHJ11U/N7IzprF9+ZbZMNDR4vVmtctd1qGtnZjy13m86phyKj6NvReSib+dQ50HAiVJkhj3+ym2nk4sM792Da1o7mROi7rmpZbrDe/aLA+J45ejV2nvUquUHARBeFzVuobozp07rF+/njlz5tCuXTucnZ2ZMmUKzs7O/Pjjj3K6f//9l3HjxrF69Wq0tdWrz6OiotixYwdLly6lZcuWtG3bloULF7JmzRquX7/+rG/puTFv3jw8PDwwMDDAwcGBd999V61WrbhJaOvWrbi4uKCvr88bb7xBTk4OK1euxMnJCTMzM8aPH68WlKxatQofHx+MjIywsbHhzTffJDk5+aFl+fTTT5k+fTqtW7emfv36TJgwgS5durBhw4aHnmdlZYWNjY382rp1K/Xr18fPz08tXWJiIl27dkVPT4969erx559/PvL5bNu2jYYNG6Knp0eHDh2Ii4tTO/5gk9mUKVNo2rQpS5cuVVtsNC0tjeHDh2NlZYWxsTEvv/wykZGRann99ddfNG/eHF1dXSwtLenVqxegavK8evUq7733nlz79aLQ1FDgaGGgFuBUlJ62JkpN1TPp6Pr0gofBrZ1YNtiH5UOa066hFZIEPx68TPJ98ycBnE/MUAuGAhpZ800fT355uwWgar5r6mD60GsNbKVqNtt3IRmfGXto+Nl2jpcyGk0QhIqr1gFRQUEBhYWFJVaq1tPT4/DhwwAUFRUxcOBAJk6cSKNGjUrkcfToUUxNTfHx8ZH3+fv7o6GhwfHjx0u9bm5uLhkZGWqvipAkifzcwip5ladDZ3loaGiwYMECzp07x8qVK9m3bx+TJk1SS5OTk8OCBQtYs2YNO3bs4MCBA/Tq1Ytt27axbds2Vq1axU8//aQWYOTn5zN9+nQiIyPZtGkTcXFxDBkypMLlS09Px9y85F/RZcnLy+PXX3/l7bffLhE4fPHFF/Tu3ZvIyEgCAwPp378/UVFRZeaVkJDA66+/To8ePYiIiGD48OF8/PHHjyzD5cuXWb9+PRs2bCAiIgKAPn36kJyczPbt2wkPD6dZs2Z07NiRlJQUAP7++2969epFt27dOHXqFHv37qVFC9UX6IYNG6hduzbTpk0jMTGRxMSyax5qIoVCgZmBkl7N7Bn+FGeTVigUdHSzxlRfyZD/+vn8HppAi6/28vXOC7T8ag8n41PluY2KBfk35A3v2rzUwJK5fTxZMsgH7UfMul3PypB2/3XkvpWVS15hEUOWl96FQBCEiqnWTWZGRkb4+voyffp03NzcsLa25vfff+fo0aM4OzsDMHv2bLS0tBg/fnypeSQlJVGrlvpfh1paWpibm5OUlFTqObNmzWLq1KmPXe6CvCL+N+HgY5//JEZ+54e2zuOv71SsuLM1qPqpzJgxg1GjRvHDDz/I+/Pz8/nxxx+pX1/V1+KNN95g1apV3LhxA0NDQ9zd3enQoQP79++nX79+ALz99tvy+fXq1WPBggU0b96crKwsDA0Ny1W2tWvXEhYWxk8//VTu+9m0aRNpaWmlBl99+vRh+PDhAEyfPp3du3ezcOFCtXu9X/E9z507FwAXFxfOnDnD7NmzH1qGvLw8fvnlF6ysVF9ohw8fJjQ0lOTkZHR0VDUh33zzDZs2beLPP/9k5MiRzJw5k/79+6v9f/T09ATA3NwcTU1NubZNKElbU4OxHeqj+4yWvWjfsBaaGgoK/+v0vGh/DAAjfwnH3uxeE2pDa0NcbVTNfAqFgt7e5WuiA1Ut0T/3TQZ5J7+Qr3deYGibulg+QY2aINR01bqGCFRNLJIkYW9vj46ODgsWLGDAgAFoaGgQHh7Od999x4oVKyq1ueCTTz4hPT1dfiUk1LzJ0Pbs2UPHjh2xt7fHyMiIgQMHcvv2bXJycuQ0+vr6cjAEYG1tjZOTk1pgY21trdYkFh4eTo8ePahTpw5GRkZy81V8vGo5hUaNGmFoaIihoSFdu3YtUa79+/czdOhQlixZItcIHjp0SD7H0NCw1A7Xy5Yto2vXrqX2MfP19S2xXVxD1LVrVznf4utFRUXRsmXLh+ZRGkdHRzkYAoiMjCQrKwsLCwu18sfGxhITo/oijYiIoGPHjo/MW6geNDQUjHvZucT+W1m5RCakAfDnKF/+HN36sX9nvexaC686pmr7Fu2PYdCy0MfKTxAElWpdQwRQv359Dh48SHZ2NhkZGdja2tKvXz/q1avHoUOHSE5Opk6de0NlCwsL+eCDD/j222+Ji4vDxsamRB+VgoICUlJSyvyrWkdHR/6L/XFoKTUY+Z3foxM+BVrKJ49x4+Li6N69O6NHj2bmzJmYm5tz+PBhhg0bRl5eHvr6qs6iD/bXUigUpe4r+m8ZhezsbAICAggICGD16tVYWVkRHx9PQEAAeXmqkUDbtm0jP1818dyDnZIPHjxIjx49mD9/PoMGDZL3+/j4yE1QoArC7nf16lX27NnzyD5HpVm6dCl37twp9X4rysBAfYRTVlYWtra2HDhwoETa4v5HD+uYLVRPYzs4Y2mow5Qt50rMjg3g7Wj2RH/AaWoo2PhuGwBCY1Po+5NqyP75xAyiEjNwszV+7LwFoSar9gFRMQMDAwwMDEhNTWXnzp3MmTOH3r174+/vr5YuICCAgQMHMnToUED1l3taWhrh4eF4e3sDsG/fPoqKikr8lV9ZFApFpTRbVZXw8HCKioqYO3cuGhqqAGvt2rVPnO+FCxe4ffs2wcHBODiolj04ceKEWhpHR8fSTuXAgQN0796d2bNnM3LkSLVjenp6chNqaZYvX06tWrV45ZVXSj1+7NgxtQDr2LFjeHl5AaiNZCzm5ubGli1bSuRRUc2aNSMpKQktLS2cnJxKTdOkSRP27t0r/39+kFKpVOu0LlQ9LU0N3mrliKWhDhP/jKSzu408OeTkHu6VWpvd3MmM5k5m8hpny0NimfOGZ6XlLwg1SbUPiHbu3IkkSbi4uHD58mUmTpyIq6srQ4cORVtbGwsLC7X02tra2NjY4OLiAqi+vLp06cKIESNYvHgx+fn5jB07lv79+5fafCKAs7Mz+fn5LFy4kB49ehASEsLixYufON86deqgVCpZuHAho0aN4uzZs0yfPv2R5+3fv5/u3bszYcIEevfuLff9UiqVj+xYXVRUxPLlyxk8eDBaWqX/d1+3bh0+Pj60bduW1atXExoayrJly8rMc9SoUcydO5eJEycyfPhwwsPDWbFixSPv40H+/v74+vry2muvMWfOHBo2bMj169fljtQ+Pj5MnjyZjh07Ur9+ffr3709BQQHbtm3jo48+AlT9u/755x/69++Pjo4OlpYll5gQqkaXxjZ0aayqhX7Nyw7nWobYmlRujZ9CoWDF0BYcuHiTMb+dZO2Ja2w/k8S8fk15qYElutrP7x9mgvCsVfs+ROnp6YwZMwZXV1cGDRpE27Zt2blzZ4WaL1avXo2rqysdO3akW7dutG3blv/9739PsdTPN09PT+bNm8fs2bNp3Lgxq1evZtasWU+cr5WVFStWrGDdunW4u7sTHBzMN99888jzVq5cSU5ODrNmzcLW1lZ+vf766488d8+ePcTHx6t15n7Q1KlTWbNmDU2aNOGXX37h999/x93dvcz0derUYf369WzatAlPT08WL17MV1999ciyPEihULBt2zbatWvH0KFDadiwIf379+fq1atys1/79u1Zt24dW7ZsoWnTprz88suEht7rKzJt2jTi4uKoX7++Wv8koXp5qYFVpQdDxQx0tOjmYUOT2qp5ljJzCxjxywneXxvxVK4nCC8qhVRZ47RfYBkZGZiYmJCeno6xsXr7/N27d4mNjVWbW0YQhKpX0z6ba08kMOnP02r7DnzY/pGzcz8LyZl3mbE1ije8a8vTBhQ7duU283ZF8+krbmw7k0hAI2u8Hcs/pYYgPMzDvr8fVO1riARBEIRHe62pPW88MHz/12NXq6g091y9nU2LmXvZEnmdQT+HUlBYpHb8+32XCY1Lof//jvK/f64w/veIEmkE4VkQAZEgCMILQKmlwTd9PDk0qYMcGK09kcCdvKrtdL8+/Jra9p6oe4vtLtp/WV4M926+Kgj6N+2OWhpBeFZEQCQIgvACcTDXZ3bvJjiY65Fxt4Atkf9WaXmOx6aobf9y9CpFRRLpd/L5eufFUs/5OSTuGZRMENSJgEgQBOEFo6mh4K2Wqiksfjl6VV7SJzIhjbeWHmftiQRuPLDW2tOQlVtA+FXVlAC/DW+JhgKOxNym3qfb6Dj3gJyueNZuS0MdtDQUhMamcO56+lMvnyDcTwREgiAIL6C+Pg7oaGlw7noGJ+PTkCSJoD8iOHz5FpP+PM0bi4889b46R2NuU1Ak4WihT2tnSzq535s09VaWajJW51qGbBrThtm9Pfh5iA9dPWwBWCFqiYRnTAREgiAILyAzAyU9PFVzrfX+8QgfrIsk9la2fDwh5Q7On21nyT9Xnsr1kzPvMuIX1cSr7RqoRpYN8nUqke7Dzg3R1dakX/M6NKltypDWqjSbI69zOyv3qZRNEEojAiJBEIQX1IAWDvLPG06W3pdo5rYoZmw9T3pOPuWdhaWoSOLDdZFM2XJOPichJYepf50jIiGNzLv5TPvrvJy+eKh96/oWfNHdHTN91Txy3ZvY0tldfQmlZnVM8axtQl5BEb+Hxpf/ZgXhCVX7maoFQRCEx9OsjlmJff5utfB3s+bjDWfkfUsPx7L0cCyvN7NnXt+mj8z39L/p/Pnf6LFeXvZ4Opgy8c9Ijl1JYXkpTV2+9VUrCigUCoa1rcvbbZyQJNViuA9SKBQMbVOXoD8iWHXsKu/41Udb8/H/dj8Vn4qpvpK6/83HFH0jkzWhCYx72RkzA+Vj5yu8eEQNkSAIwgtKoVCwbfxLNHe6Fxi90sSW/i3qEPy6R4n0G07+y6L9l8m8m//QfA9fuin/3HNRCF9sOsuxKymlpl0/ujWGOup/eysUilKDoWLdPGyxMtLhRkYu284kPrQspTkRl0Jqdh5Xb2fTZ/FR3vjxCFm5Bfx9OpFPNpzh55BY5u4ufYSbUHOJgEgotwMHDqBQKEhLS3vm116xYoW8AvyTcHJy4ttvv33ifOLi4lAoFERERABV+2wE4WHc7YxZ+44vLeuaY2WkQweXWgD0b1GHyMmdsTLSUUv/9c6LjPv9lLxdWCRxNOY2+fd1wC6eO6jYqjImgKxnZYC3Y8laqkdRamnIo+RKq3F6mCMxt3hj8VGGrAhj34VkCookbmfnEbjkGGN+OymPelsf/i/pdx4e+Ak1iwiIhHJr3bo1iYmJmJiYPDJtVQQI7du3R6FQlHiVtcp9ZarIs3kS7du3Jygo6KleQ3jxKBQKVg9vydGPX8ZU/14zkYmeNnve8+PXYS1xtNCX9x+4eJNeP4Rw+loaSw9dYcCSY8z8OwqAnLx7Q+kf1NTBlD73zZa9dJDPY5f5zZZ1UGpqEJGQxqn40q9Xmt3nVZM6Riak8dPBex3GI6+pD+O/k1/IuhMJ8vbJ+FS+33eJvAIxS3ZNJfoQCeWmVCqxsbF5dMIKyMvLQ6msnHb8DRs2kJeXJ2/fvn0bT09P+vTpUyn5P8yjnk1hYeF/zQTibxChamiV0Q/HRF+btg0s2fu+H53m/yOPRDsVn8ar34fI6VYciSM58y4+jubkF0rYm+oxr68nw385QebdAgD6+NQmsKUjX/RwR19bs8xrloeVkQ7dPW3ZcPJfVhyJw6uU/lClCbmv9irpv7mWNBRQdF9/cQOlJtl5haw8GsfQNnVZdTSOKf91Ak/NyWfcy85qgaNQM4jfzk+BJEnk371bJa+KrNXbvn17xo0bR1BQEGZmZlhbW7NkyRKys7MZOnQoRkZGODs7s337dqBkrc/Vq1fp0aMHZmZmGBgY0KhRI7Zt20ZcXBwdOnQAwMzMDIVCwZAhQ+Rrjh07lqCgICwtLQkICABg3rx5eHh4YGBggIODA++++y5ZWVkVeu7m5ubY2NjIr927d6Ovr18iIMrMzGTAgAEYGBhgb2/PokWLHpl3aGgoXl5e6Orq4uPjw6lTp9SOP/hsipv4tmzZgru7Ozo6OsTHx5Obm8uHH36Ivb09BgYGtGzZkgMHDqjlFRISQvv27dHX18fMzIyAgABSU1MZMmQIBw8e5LvvvpNrv+Li4ir0jAShLFqaGqwb5cvYDs5lptl2JolpW1WBw0sNLGlZz4LwzztRz9IAA6Um/m6qeYaMdbWfKBgq9nabugD8fTqRGxl3ycotIK+giFtZuVxPu1Mi/Y2Mu0TfyEKhUE1OCWBpqKSvj2q0nY6WBu/5N2TX+36Y6muTkHKH+p9uk4MhgGWHY/H7+gA3M8WQ/5pG1BA9BQW5uSwY/EaVXHv8yj/RrsDK3itXrmTSpEmEhobyxx9/MHr0aDZu3EivXr349NNPmT9/PgMHDiQ+vuTw1zFjxpCXl8c///yDgYEB58+fx9DQEAcHB9avX0/v3r25ePEixsbG6OnpqV1z9OjRhITc++tTQ0ODBQsWULduXa5cucK7777LpEmT+OGHHx77WSxbtoz+/ftjYKC+2vfXX3/Np59+ytSpU9m5cycTJkygYcOGdOrUqdR8srKy6N69O506deLXX38lNjaWCRMmPPL6OTk5zJ49m6VLl2JhYUGtWrUYO3Ys58+fZ82aNdjZ2bFx40a6dOnCmTNnaNCgAREREXTs2JG3336b7777Di0tLfbv309hYSHfffcd0dHRNG7cmGnTpgFgZWX1iFIIQvlZGuow9mVnTlxNKbOTdLHikWNKLQ02vNuaO/mFWBuX/3dPeTS2N6G5kxlhcanM3nGBXedu4FXHlLjb2aRm57PnfT9sTO5ds7h2yMPeBCcLA7ZEXuelBlaM6eDMuesZ9GvuwFutVH2T+jevw+KDMaVeN/1OPr8dj2eCf4NKvR+hehMBUQ3n6enJ559/DsAnn3xCcHAwlpaWjBgxAoAvv/ySH3/8kdOnT5c4Nz4+nt69e+PhoRqtUq9ePfmYubk5ALVq1SrRGbpBgwbMmTNHbd/9/WKcnJyYMWMGo0aNeuyAKDQ0lLNnz7Js2bISx9q0acPHH38MQMOGDQkJCWH+/PllBkS//fYbRUVFLFu2DF1dXRo1asS1a9cYPXr0Q8uQn5/PDz/8gKenJ6B6XsuXLyc+Ph47O9WEeR9++CE7duxg+fLlfPXVV8yZMwcfHx+1+27UqJH8s1KpRF9fv9KbLgWhmK62JmtG+nLmWjrf7b3EiaspvN2mLtvPJhGVmCGnK64NAjDVV2L6lMozpHVdwuJS5XmUDl261yS2JiyeIP+G8nZxZ+82zpYMb1sXB3M93mzpiL2pHn+Na6uW70BfR7WASE9bk1c97fjjv35Fv4fGM6ZD/Uqp6RKeDyIgegq0dHQYv/LPKrt2RTRp0kT+WVNTEwsLCznAAbC2Vv3SS05OxtjYWO3c8ePHM3r0aHbt2oW/vz+9e/dWy68s3t7eJfbt2bOHWbNmceHCBTIyMigoKODu3bvk5OSgr6+vljY+Ph53d3d5+9NPP+XTTz9VS7Ns2TI8PDxo0aJFiWv5+vqW2C4eeTZq1Ch+/fVX+VhWVhZRUVE0adIE3ftq3h7MozRKpVLteZw5c4bCwkIaNmyoli43NxcLC9Vf2xEREc+kz5MgPIpHbROWDr7XKXp8xwbE3MwiMiGNXl72KBRlD5uvTAGNrLEz0eV6esm119aduMa4lxugqaFAkiS5hqitsyUWhjpMDHAtM197Uz1m9/bg801nyS+UmNmrMd08bHG01GfOjoskZdxl1/kbdPtvKRHhxScCoqdAoVBUqNmqKmlra6ttKxQKtX3Fv/SKikqOvBg+fDgBAQH8/fff7Nq1i1mzZjF37lzGjRv30Gs+2IQVFxdH9+7dGT16NDNnzsTc3JzDhw8zbNgw8vLySgREdnZ28nB3uFcbVSw7O5s1a9bIzUoVMW3aND788MMKn1caPT09tS+NrKwsNDU1CQ8PR1NTUy2toaGhfI4gVFf1rQypb2X4TK+ppanBQF8nZu+4UOLYv2l36LP4CJ+94oaxrjY3MnLR0dIo91D/fs3r0NfHgRsZuVgb66BQKHi3vTN38gpZuO8yK4/EiYCoBhF1gcITcXBwYNSoUWzYsIEPPviAJUuWAMgjxwoLCx+ZR3h4OEVFRcydO5dWrVrRsGFDrl+/XmZ6LS0tnJ2d5deDAdG6devIzc3lrbfeKvX8Y8eOldh2c3MDVE189+cN4ObmxunTp7l7926ZeZSHl5cXhYWFJCcnq13D2dlZbgJr0qQJe/fuLTMPpVJZrmcqCC+SAS0csDfVw8fRDH83a8wNlPRvruoofTI+jd4/HmXvhWQAWtQ1R1db82HZqVEoFNiY6Kr98fJmyzpoaig4HpvChaSMh5wtvEhEQCQ8tqCgIHbu3ElsbCwnT55k//79cmDh6OiIQqFg69at3Lx586EjxpydncnPz2fhwoVcuXKFVatWsXjx4scu17Jly3jttdfkZqgHhYSEMGfOHKKjo1m0aBHr1q17aCfpN998E4VCwYgRIzh//jzbtm3jm2++qXC5GjZsSGBgIIMGDWLDhg3ExsYSGhrKrFmz+PvvvwFVP66wsDDeffddTp8+zYULF/jxxx+5dUvVFODk5MTx48eJi4vj1q1bpdbcCcKLxlRfyYGJ7Vk3ypf/DfTmxGf+vN9Zvel54d5LgKr/0JOyNdEjoJGqu8DKI6pJJ9Ny8th5LomiovKP5BWeLyIgEh5bYWEhY8aMwc3NjS5dutCwYUO5M7C9vT1Tp07l448/xtramrFjx5aZj6enJ/PmzWP27Nk0btyY1atXM2vWrMcq08WLF+XmtrJ88MEHnDhxAi8vL2bMmMG8efPk4f+lMTQ05K+//uLMmTN4eXnx2WefMXv27Mcq3/Llyxk0aBAffPABLi4uvPbaa4SFhVGnTh1AFTTt2rWLyMhIWrRoga+vL5s3b0ZLS9W6/eGHH6KpqYm7uztWVlaljv4ThBeRtqaGvOSHhoaCWka6jOlQXz6enaeqOW1bCQERwCBfJwA2nfqXjLv5TP3rPO+sCqfep9uYu0ss+/EiUkgVmbimhsrIyMDExIT09PQSHYvv3r1LbGwsdevWVet0KwhC1RKfzRefJElk3C3Ad9ZecvIKMdPXJvzzTg9dJ60ieQd8+w/RN7IY/7IzC/ZdVjt+7JOOakP+herpYd/fDxI1RIIgCMJzSaFQYKKnLU+82LaBVaUEQ8V5D/xvzqIHgyFQzdwtvFhEQCQIgiA81yYGuPBJV1c+7Vb2MPvH8Xqz2iX2WRqqpjb57fhVsnMLKvV6QtUSAZEgCILwXDPQ0eIdv/rYmlTutBUGOlry8iEAf4xsReinHalraUDG3QLW/jeJ49xdFxn8cyhZIkB6romASBAEQRDK8Ek3VxYM8CLyy860rGeBhoaCt9uqgqRlh2NJy8njhwMxHIy+yZpQMcjheSYCIkEQBEEog7amBq962mGif2/C2jea1cbcQMm11Dt8sfkchf8NxZ+5LYpF+y+Tk1dQoYW2hepBBESCIAiCUAF6Sk25w/VfkfcmkZUk+HrnRdy/3Mnnm85WVfGExyQCIkEQBEGooEG+juho3fsKbVLbRO346uPxxN7KBqCwSGLH2USW/HNFTOxYjYmASBAEQRAqyMJQhze8VaPQlJoaLOjvhZGu+vKgyw5fAeCj9acZ9etJZm6LYv3Ja8+8rEL5iIBIeOrat29PUFCQvO3k5CSvLv+8iIuLQ6FQqC0qKwhCzTbKrz42xrq84VMbJ0sD9rzvx8Z3W/NaUzsA1p24RlL6Xf4MvxcE/XrsalUVV3gEERAJz1xYWBgjR46UtxUKBZs2baq6Av1XhtJeX3/9dZWWqzpYsWIFpqamVV0MQah2HMz1OfZpR77q5QGAtbEuXnXMmN+vKR72JuQWFPH+2gi1cyKvpXMyPrUKSis8igiIhGfOysoKfX39Cp2Tl5f3lEqjkpiYqPb6+eefUSgU9O7d+6letyo97Wf6oMLCQrEYrVAjKBQKRrSrB8CRmNsAdGlkw+vN7AFY8s8V5u26yLAVYaTlPNvPoVA2ERDVYO3bt2fcuHEEBQVhZmaGtbU1S5YsITs7m6FDh2JkZISzszPbt2+Xzzl79ixdu3bF0NAQa2trBg4cKK/EDpCdnc2gQYMwNDTE1taWuXPnlrju/U1mTk5OAPTq1QuFQiFvT5kyhaZNm7J06VK1taji4+Pp2bMnhoaGGBsb07dvX27cuCHnXXzeqlWrcHJywsTEhP79+5OZmfnQZ2FjY6P22rx5Mx06dKBevXpq6S5cuEDr1q3R1dWlcePGHDx48JHPecmSJTg4OKCvr0+vXr2YN29eiRqXzZs306xZM3R1dalXrx5Tp06loODeJG8KhYKlS5fSq1cv9PX1adCgAVu2bFHL41HvTfv27Rk7dixBQUFYWlrKC9rOmzcPDw8PDAwMcHBw4N133yUrKwuAAwcOMHToUNLT0+VasylTpgCQmprKoEGDMDMzQ19fn65du3Lp0iX5esU1S1u2bMHd3R0dHR2xGK1QY3RrbIO96b2JIv3drRneVvX7ZPvZJBbsu8zeC8nM2x1dVUUUHiACoqdAkiSK8gqr5FXRuS9WrlyJpaUloaGhjBs3jtGjR9OnTx9at27NyZMn6dy5MwMHDiQnJ4e0tDRefvllvLy8OHHiBDt27ODGjRv07dtXzm/ixIkcPHiQzZs3s2vXLg4cOMDJkyfLvH5YWBigWgU+MTFR3ga4fPky69evZ8OGDURERFBUVETPnj1JSUnh4MGD7N69mytXrtCvXz+1PGNiYti0aRNbt25l69atHDx4kODg4HI/kxs3bvD3338zbNiwEscmTpzIBx98wKlTp/D19aVHjx7cvn27zLxCQkIYNWoUEyZMICIigk6dOjFz5ky1NIcOHWLQoEFMmDCB8+fP89NPP7FixYoS6aZOnUrfvn05ffo03bp1IzAwkJSUFIByvTeger+VSiUhISEsXrwYAA0NDRYsWMC5c+dYuXIl+/btY9KkSQC0bt2ab7/9FmNjY7n27MMPPwRgyJAhnDhxgi1btnD06FEkSaJbt27k5+fL18vJyWH27NksXbqUc+fOUatWrfK+DYLwXNPS1JAncARo62yJu50xLeuaq6XbEnmdu/mFT3y941duszfqxqMTCmWThEdKT0+XACk9Pb3EsTt37kjnz5+X7ty5I+8rzC2QEj76p0pehbkF5b4vPz8/qW3btvJ2QUGBZGBgIA0cOFDel5iYKAHS0aNHpenTp0udO3dWyyMhIUECpIsXL0qZmZmSUqmU1q5dKx+/ffu2pKenJ02YMEHe5+joKM2fP1/eBqSNGzeq5Tt58mRJW1tbSk5Olvft2rVL0tTUlOLj4+V9586dkwApNDRUPk9fX1/KyMiQ00ycOFFq2bJluZ/L7NmzJTMzM7X3NDY2VgKk4OBgeV9+fr5Uu3Ztafbs2WXm1a9fP+mVV15R2xcYGCiZmJjI2x07dpS++uortTSrVq2SbG1t5W1A+vzzz+XtrKwsCZC2b98uSZL0yPdGklTvt5eX16NuX1q3bp1kYWEhby9fvlytvJIkSdHR0RIghYSEyPtu3bol6enpye//8uXLJUCKiIh45DWfhtI+m4LwLGXezZdeW3RYClpzSt73V+S/kuNHW9Vevx2/+kTXyc7Nl/OKTEh9skK/YB72/f0gUUNUwzVp0kT+WVNTEwsLCzw8POR91tbWACQnJxMZGcn+/fsxNDSUX66uqsUUY2JiiImJIS8vj5YtW8rnm5ub4+Li8lhlc3R0xMrKSt6OiorCwcEBBwcHeZ+7uzumpqZERUXJ+5ycnDAyMpK3bW1tSU5OBmD16tVq5T906FCJ6/78888EBgbKzXT38/X1lX/W0tLCx8dHvnajRo3kfLt27QrAxYsXadGihVoeD25HRkYybdo0tXKNGDGCxMREcnJy5HT3v1cGBgYYGxvL9/Wo96aYt7d3iXvas2cPHTt2xN7eHiMjIwYOHMjt27fVrv2gqKgotLS01N5rCwsLXFxc1N4LpVKpVm5BqEkMdbTY+G4b5vdrKu97xcOWqa82YuXbLfj8FTcAlh6q2PxE4VdT+CvyOsev3KaoSGL/hZvysWWHY0s95+y/6fT8/jDbzyQ+3s3UAFqPTiJUlEJbA7tpravs2hWhra2ttq1QKNT2KRQKAIqKisjKyqJHjx7Mnj27RD62trZcvnz5MUpcNgMDg8c6r7R7Ku7M++qrr6p9idvb26ulPXToEBcvXuSPP/6o8HW3bdsmNxfp6ZV/kcmsrCymTp3K66+/XuLY/UHZw+7rUe9NsQefaVxcHN27d2f06NHMnDkTc3NzDh8+zLBhw8jLy6tw5/cH6enpyf+HBEFQfW4Ht3YCoFkdU77bc4mYm9mMXHWCPVHJ+LtZM7p9PbwdzUs9P7egkN4/Hi0z/79PJ/JpNzesje/97lh17Cpf/Ddz9ujVJ7k4ows6WpqVd1MvCBEQPQUKhQKF8sX7z9asWTPWr1+Pk5MTWlol/+vUr18fbW1tjh8/Tp06dQBVx9vo6Gj8/PzKzFdbW5vCwke3obu5uZGQkEBCQoJcS3T+/HnS0tJwd3cv1z0YGRmp1R49aNmyZXh7e+Pp6Vnq8WPHjtGuXTsACgoKCA8PZ+zYsYCqRutBLi4uav2igBLbzZo14+LFizg7O5frHkrzqPemLOHh4RQVFTF37lw0NFTB9Nq1a9XSKJXKEu+Pm5sbBQUFHD9+nNatVcH/7du3uXjxYrnfC0Go6Yx0tenX3IGlh2PZE6Wq7d0TdYM9UTfYOq4tje1N+O14PFGJGXzR3R2llgbhcQ8fsl9QJLEmNIEJ/g0AOH89Qw6Gim0+dZ2+zR1KO71GE01mQrmNGTOGlJQUBgwYQFhYGDExMezcuZOhQ4dSWFiIoaEhw4YNY+LEiezbt4+zZ88yZMgQ+Yu2LE5OTuzdu5ekpCRSU8v+sPv7++Ph4UFgYCAnT54kNDSUQYMG4efnh4+PzxPfX0ZGBuvWrWP48OFlplm0aBEbN27kwoULjBkzhtTUVN5+++0y048bN45t27Yxb948Ll26xE8//cT27dvVak2+/PJLfvnlF6ZOncq5c+eIiopizZo1fP755+Uu+6Pem7I4OzuTn5/PwoULuXLlCqtWrZI7WxdzcnIiKyuLvXv3cuvWLXJycmjQoAE9e/ZkxIgRHD58mMjISN566y3s7e3p2bNnucstCDVdcW3Rg77dE82Za+lM+escq45dZeKfkZyMT+XgpZulpgeY3VvV3eG30KvkF6pqj0vraP2/CjbR1RQiIBLKzc7OjpCQEAoLC+ncuTMeHh4EBQVhamoqBz1ff/01L730Ej169MDf35+2bduW2m/lfnPnzmX37t04ODjg5eVVZjqFQsHmzZsxMzOjXbt2+Pv7U69evcdq3irNmjVrkCSJAQMGlJkmODiY4OBgPD09OXz4MFu2bMHS0rLM9G3atGHx4sXMmzcPT09PduzYwXvvvafWFBYQEMDWrVvZtWsXzZs3p1WrVsyfP7/UGqeylOe9KY2npyfz5s1j9uzZNG7cmNWrVzNr1iy1NK1bt2bUqFH069cPKysr5syZA6hGBnp7e9O9e3d8fX2RJIlt27aVaNoTBKFsDub68szW99sTlUyP7w+TV6AKbDZHXOf1H46w/UwSAL2b1eYN79oY6Wgx/mVnYmd1o5dXbSwNldzIyKXn9yEUFBYRGqcaiWqko0X45/4Y6mhxOTmLg9FlB1Y1lUKSKjhOuwbKyMjAxMSE9PR0jI2N1Y7dvXuX2NhYtblyBOFhRowYwYULF0rt0C1UHvHZFJ4Xd/MLOXc9Ay8HU3ZH3eD7fZc58296mekVCgj/vBPmBsoSx77ZeZHv96v6cwa/7sHUv85zJ7+QnUHtcLExYubf51lyKJZW9cxZM9K3xPkvmod9fz9I1BAJwlP2zTffEBkZyeXLl1m4cCErV65k8ODBVV0sQRCqCV1tTbwdzdDQUBDQyIZPu7mpHXcwVx+k0cTepNRgCNSb4D7ecIY7+YVYGChpaG0IwNA2ddHSUHDsSgqnr6VV6n0870RAJAhPWWhoKJ06dcLDw4PFixezYMGCh/ZTEgShZvOtb8HKt1vwWTc3xnZwZs/7fiwZ5INSU/WV7dfQqsxzrYx0CPvMHy2Ne/0UW9W3kPst2pnq0cNT1US35FDpQ/RrqmofEGVmZhIUFISjoyN6enq0bt1aHqWTn5/PRx99JC87YGdnx6BBg7h+/bpaHikpKQQGBmJsbIypqSnDhg2TlyYQhKdt7dq1JCcnc+fOHc6dO8eoUaOqukiCIFRzfg2tGNGuHh8GuKCjpUknd2vGveyMvakerzer/dBzrYx0CGhsI2/71rNQOz78JdUM2n9FXuf1H0KISEir9PI/j6p9QDR8+HB2797NqlWrOHPmDJ07d8bf359///2XnJwcTp48yRdffMHJkyfZsGEDFy9e5NVXX1XLIzAwkHPnzrF79262bt3KP//8o7bauiAIgiBUd+M6NiDk45dxsnz0HG1D72s6862vHhA1sjOhrbNqMMjJ+DReWxTy1EedJabf4WT8w6cMqGrVulP1nTt3MDIyYvPmzbzyyivyfm9vb7p27cqMGTNKnBMWFkaLFi24evUqderUISoqCnd3d8LCwuSh2Tt27KBbt25cu3YNO7uSvfsfJDpVC8LzR3w2hZpu4d5LFEoSEzo2KDFB6j/RNxn0c6i8vXSQD/7u1k+lHJduZNJp/j8ArB7ekjbOZY/MrWwvTKfqgoICCgsLS/wy09PT4/Dhw6WeU7wqd/Fq4kePHsXU1FRtnhp/f380NDQ4fvx4qXnk5uaSkZGh9hIEQRCE58m4jg0I8m9Y6mzxLzWwVGtKW3wwpkSayrA36oYcDAHM3x39VK5TGap1QGRkZISvry/Tp0/n+vXrFBYW8uuvv3L06FESE0uux3L37l0++ugjBgwYIEeCSUlJJVbY1tLSwtzcnKSkpFKvO2vWLExMTOTX/WtnCYIgCMLzTqFQsGpYC458/DJKTQ1OXE0l7L85i4pF38gkNTuv1PMlSeJmZu4jrzNs5Qm17RNXU4m5WT378FbrgAhg1apVSJKEvb09Ojo6LFiwgAEDBpSYbC4/P5++ffsiSRI//vjjE13zk08+IT09XX4lJCQ8UX6CIAiCUN1oaWpgZ6pHb29VJ+0fD9yrJbqQlEHX7w4RuPQ4sbeyee+PCC4nqwKZoiKJ0b+epPnMPfx2PL7M/FMeCKZcbVTLJq0IiavkO6kc1X4ts/r163Pw4EGys7PJyMjA1taWfv36Ua9ePTlNcTB09epV9u3bp9ZOaGNjI68IXqygoICUlBRsbGwojY6ODjo6Ok/nhgRBEAShGnmnXT3+CItn34VkLiRl4GpjzPYzSRQWSZxPzKD//45yIyOXf9PusPYdX1aHxrPjnKqF5dONZ+ja2IbF/8Sg1NRg4b7LGCg1eauVIw2sVQGQvake+z9sz4mrKby55Dh/hl/jg84NMdUvfS6lqlLta4iKGRgYYGtrS2pqKjt37pTXSyoOhi5dusSePXuwsFDvTe/r60taWhrh4eHyvn379lFUVKS26rlQfTk5OfHtt9/K2wqFgk2bNlVZeQRBEF4kTpYGdG1sC0CXbw/Rc1EI3+29JB+/kaFqGguNTSEsLoXFB9T7G3Wa/w8/HbzCwn2qGbKz8wr56Z8rfLguEoBuHjYotTTwrWeBq40Rd/IL+T20+rW8VPuAaOfOnezYsYPY2Fh2795Nhw4dcHV1ZejQoeTn5/PGG29w4sQJVq9eTWFhIUlJSSQlJZGXp6qqc3Nzo0uXLowYMYLQ0FBCQkIYO3Ys/fv3L9cIM+HRSgtQDh8+TJs2bbCwsEBPTw9XV1fmz59fKddLTEyka9eulZJXWaZMmULTpk2f6jUEQRCqi1F+9eWfIx8yL1GfxUf5N+2O2r5bWQ/vS9Tuv4kkFQoFw19Ste7M3nGBtrP3kZCS85glrnzVPiBKT09nzJgxuLq6MmjQINq2bcvOnTvR1tbm33//ZcuWLVy7do2mTZtia2srv44cOSLnsXr1alxdXenYsSPdunWjbdu2/O9//6vCu3rxGRgYMHbsWP755x+ioqL4/PPP+fzzzyvludvY2Dy0STM/P/+JryEIglCTeNQ2wUy/5MLMXnVMAdDT1uTBwWpnpwag1Hp4GKGrrUFzJ3N5u4enLRb/LTtyLfUOn24882QFr0TVPiDq27cvMTEx5ObmkpiYyPfff4+JiQmgakqRJKnUV/v27eU8zM3N+e2338jMzCQ9PZ2ff/4ZQ0PDKrqj6iUzM5PAwEC5SXL+/Pm0b9+eoKAgQPWMp0+fzoABAzAwMMDe3p5FixbJ5zs5OQHQq1cvFAqFvO3l5cWAAQNo1KgRTk5OvPXWWwQEBDxyQdPk5GR69OiBnp4edevWZfXq1SXS3F8jFRcXh0Kh4I8//sDPzw9dXV35nKVLl+Lm5oauri6urq788MMPavlcu3aNAQMGYG5ujoGBAT4+Phw/fpwVK1YwdepUIiMjUSgUKBQKVqxYUfGHKwiC8BxZMbQFhjqqrsUKBYR8/DI/veVNPx8H/ninFc0d7wU2A1s5YqijRdf7ZsReP7o10TO6Ehf8Cn19VB21W9a1QFdbU06jo6VJz6b28vaRmNskZ9x92rdWLtW+U/XzSJKkKqul0NbWLnXOibK8//77hISEsGXLFqytrfnyyy85efKkWnPR119/zaeffsrUqVPZuXMnEyZMoGHDhnTq1ImwsDBq1arF8uXL6dKlC5qamqVe59SpUxw5cqTUyTTvN2TIEK5fv87+/fvR1tZm/PjxJTrFl+bjjz9m7ty5eHl5yUHRl19+yffff4+XlxenTp1ixIgRGBgYMHjwYLKysvDz88Pe3p4tW7ZgY2PDyZMnKSoqol+/fpw9e5YdO3awZ88eADkIFwRBeFF5OphydmoAd/ML0VAo5Nqf2W80AWBWbw/6LD6Kbz0LJvdwB+CL7u5k3i2gexNbvB3N5Lw+DHBBQ6FgkK9TietM8G9AfEoOe6JuUFgksSwklk+6upVI96yJgOgpyM/P56uvvqqSa3/66acoleXruZ+ZmcnKlSv57bff6NixIwDLly8v0beqTZs2fPzxxwA0bNiQkJAQ5s+fT6dOnbCyUrUNm5qaljpqr3bt2ty8eZOCggKmTJny0EVNo6Oj2b59O6GhoTRv3hyAZcuW4eb26A9KUFAQr7/+urw9efJk5s6dK++rW7cu58+f56effmLw4MH89ttv3Lx5k7CwMMzNVX/1ODs7y+cbGhqipaVV5khEQRCEF9X9NTr3q29lSPjn/mp/dFsa6vDzkOYl0tYy0iW4d5NS8zHR02bpYB92nUti5Kpwfj8ez4SODdBXVm1IUu2bzISn58qVK+Tn59OiRQt5n4mJCS4uLmrpfH19S2xHRUWV6xqHDh3ixIkTLF68mG+//Zbff/8dUPXrMjQ0lF+HDh0iKioKLS0tvL295fNdXV3lWccf5v6ZyLOzs4mJiWHYsGFq15gxYwYxMarREREREXh5ecnBkCAIgvBoFWmBeJSObtbUMdcn424BG07+W2n5Pi5RQ/QUaGtr8+mnn1bZtauTunVVqyp7eHhw48YNpkyZwoABA3j11VfVpj2wt7dn165dj30dA4N7ix1mZakmD1uyZEmJqRWKm/T09PQe+1qCIAjCk9PUUDCktRPTtp7n55BY3mxRBw2Nygu4KkoERE+BQqEod7NVVapXrx7a2tqEhYVRp04dQDWqLzo6mnbt2snpjh07pnbesWPH1JqxtLW1KSwsfOT1ioqKyM1VDc80MjLCyMhI7birqysFBQWEh4fLTWYXL14kLS2tQvdlbW2NnZ0dV65cITAwsNQ0TZo0YenSpaSkpJRaS6RUKst1T4IgCMLj6+NTm/l7orlyM5ud55Lo6mFbZWURTWY1mJGREYMHD2bixIns37+fc+fOMWzYMDQ0NNSqRUNCQpgzZw7R0dEsWrSIdevWMWHCBPm4k5MTe/fuJSkpidTUVAAWLVrEX3/9xaVLl7h06RLLli3jm2++4a233iqzPC4uLnTp0oV33nmH48ePEx4ezvDhwx+rNmfq1KnMmjWLBQsWEB0dzZkzZ1i+fDnz5s0DYMCAAdjY2PDaa68REhLClStXWL9+PUePHpXvKTY2loiICG7duiUHcoIgCELlMdLVZkhrJwAW7ruMJElVVhYRENVw8+bNw9fXl+7du+Pv70+bNm3koerFPvjgA06cOIGXlxczZsxg3rx5BAQEyMfnzp3L7t27cXBwwMvLC1DVBn3yySc0bdoUHx8fFi1axOzZs5k2bdpDy1PcqdvPz4/XX3+dkSNHllictzyGDx/O0qVLWb58OR4eHvj5+bFixQq5CU+pVLJr1y5q1apFt27d8PDwIDg4WG5S6927N126dKFDhw5YWVnJfZ8EQRCEyvV2m7oYKDXRV2qSmlN188gppKoMx54TGRkZmJiYkJ6errZOGsDdu3eJjY2lbt26akHE8yo7Oxt7e3vmzp3LsGHDcHJyIigoSJ6XSBCeFy/aZ1MQXmQJKTnUNtOr1E7b8PDv7weJPkQ13KlTp7hw4QItWrQgPT1drsEpXitOEARBEJ42B3P9qi6CCIgE+Oabb7h48SJKpRJvb28OHTqEpaVlVRdLEARBEJ4ZERDVcF5eXoSHh5d5PC4u7tkVRhAEQRCqiOhULQiCIAhCjScCIkEQBEEQajwREFUSMVhPEKoX8ZkUBKEiRED0hIqXysjJyanikgiCcL/iz2R1W85GEITqSXSqfkKampqYmpqSnJwMgL6+fqXPoyAIQvlJkkROTg7JycmYmprKk20KgiA8jAiIKoGNjQ2AHBQJglD1TE1N5c+mIAjCo4iAqBIoFApsbW2pVasW+flVN+24IAgq2traomZIEIQKEQFRJdLU1BS/hAVBEAThOSQ6VQuCIAiCUOOJgEgQBEEQhBpPBESCIAiCINR4og9RORRP8JaRkVHFJREEQRAEobyKv7fLM1GrCIjKITMzEwAHB4cqLokgCIIgCBWVmZmJiYnJQ9MoJDG//SMVFRVx/fp1jIyMKn3SxYyMDBwcHEhISMDY2LhS8xbuEc/52RDP+dkQz/nZEc/62Xhaz1mSJDIzM7Gzs0ND4+G9hEQNUTloaGhQu3btp3oNY2Nj8WF7BsRzfjbEc342xHN+dsSzfjaexnN+VM1QMdGpWhAEQRCEGk8ERIIgCIIg1HgiIKpiOjo6TJ48GR0dnaouygtNPOdnQzznZ0M852dHPOtnozo8Z9GpWhAEQRCEGk/UEAmCIAiCUOOJgEgQBEEQhBpPBESCIAiCINR4IiASBEEQBKHGEwFRFVq0aBFOTk7o6urSsmVLQkNDq7pIz5VZs2bRvHlzjIyMqFWrFq+99hoXL15US3P37l3GjBmDhYUFhoaG9O7dmxs3bqiliY+P55VXXkFfX59atWoxceJECgoKnuWtPFeCg4NRKBQEBQXJ+8Rzrhz//vsvb731FhYWFujp6eHh4cGJEyfk45Ik8eWXX2Jra4uenh7+/v5cunRJLY+UlBQCAwMxNjbG1NSUYcOGkZWV9axvpdoqLCzkiy++oG7duujp6VG/fn2mT5+uttaVeM6P559//qFHjx7Y2dmhUCjYtGmT2vHKeq6nT5/mpZdeQldXFwcHB+bMmVM5NyAJVWLNmjWSUqmUfv75Z+ncuXPSiBEjJFNTU+nGjRtVXbTnRkBAgLR8+XLp7NmzUkREhNStWzepTp06UlZWlpxm1KhRkoODg7R3717pxIkTUqtWraTWrVvLxwsKCqTGjRtL/v7+0qlTp6Rt27ZJlpaW0ieffFIVt1TthYaGSk5OTlKTJk2kCRMmyPvFc35yKSkpkqOjozRkyBDp+PHj0pUrV6SdO3dKly9fltMEBwdLJiYm0qZNm6TIyEjp1VdflerWrSvduXNHTtOlSxfJ09NTOnbsmHTo0CHJ2dlZGjBgQFXcUrU0c+ZMycLCQtq6dasUGxsrrVu3TjI0NJS+++47OY14zo9n27Zt0meffSZt2LBBAqSNGzeqHa+M55qeni5ZW1tLgYGB0tmzZ6Xff/9d0tPTk3766acnLr8IiKpIixYtpDFjxsjbhYWFkp2dnTRr1qwqLNXzLTk5WQKkgwcPSpIkSWlpaZK2tra0bt06OU1UVJQESEePHpUkSfUB1tDQkJKSkuQ0P/74o2RsbCzl5uY+2xuo5jIzM6UGDRpIu3fvlvz8/OSASDznyvHRRx9Jbdu2LfN4UVGRZGNjI3399dfyvrS0NElHR0f6/fffJUmSpPPnz0uAFBYWJqfZvn27pFAopH///ffpFf458sorr0hvv/222r7XX39dCgwMlCRJPOfK8mBAVFnP9YcffpDMzMzUfm989NFHkouLyxOXWTSZVYG8vDzCw8Px9/eX92loaODv78/Ro0ersGTPt/T0dADMzc0BCA8PJz8/X+05u7q6UqdOHfk5Hz16FA8PD6ytreU0AQEBZGRkcO7cuWdY+upvzJgxvPLKK2rPE8RzrixbtmzBx8eHPn36UKtWLby8vFiyZIl8PDY2lqSkJLXnbGJiQsuWLdWes6mpKT4+PnIaf39/NDQ0OH78+LO7mWqsdevW7N27l+joaAAiIyM5fPgwXbt2BcRzfloq67kePXqUdu3aoVQq5TQBAQFcvHiR1NTUJyqjWNy1Cty6dYvCwkK1LwcAa2trLly4UEWler4VFRURFBREmzZtaNy4MQBJSUkolUpMTU3V0lpbW5OUlCSnKe19KD4mqKxZs4aTJ08SFhZW4ph4zpXjypUr/Pjjj7z//vt8+umnhIWFMX78eJRKJYMHD5afU2nP8f7nXKtWLbXjWlpamJubi+f8n48//piMjAxcXV3R1NSksLCQmTNnEhgYCCCe81NSWc81KSmJunXrlsij+JiZmdljl1EERMILYcyYMZw9e5bDhw9XdVFeOAkJCUyYMIHdu3ejq6tb1cV5YRUVFeHj48NXX30FgJeXF2fPnmXx4sUMHjy4ikv34li7di2rV6/mt99+o1GjRkRERBAUFISdnZ14zjWcaDKrApaWlmhqapYYhXPjxg1sbGyqqFTPr7Fjx7J161b2799P7dq15f02Njbk5eWRlpamlv7+52xjY1Pq+1B8TFA1iSUnJ9OsWTO0tLTQ0tLi4MGDLFiwAC0tLaytrcVzrgS2tra4u7ur7XNzcyM+Ph6495we9nvDxsaG5ORkteMFBQWkpKSI5/yfiRMn8vHHH9O/f388PDwYOHAg7733HrNmzQLEc35aKuu5Ps3fJSIgqgJKpRJvb2/27t0r7ysqKmLv3r34+vpWYcmeL5IkMXbsWDZu3Mi+fftKVKN6e3ujra2t9pwvXrxIfHy8/Jx9fX05c+aM2odw9+7dGBsbl/hyqqk6duzImTNniIiIkF8+Pj4EBgbKP4vn/OTatGlTYtqI6OhoHB0dAahbty42NjZqzzkjI4Pjx4+rPee0tDTCw8PlNPv27aOoqIiWLVs+g7uo/nJyctDQUP/q09TUpKioCBDP+WmprOfq6+vLP//8Q35+vpxm9+7duLi4PFFzGSCG3VeVNWvWSDo6OtKKFSuk8+fPSyNHjpRMTU3VRuEIDzd69GjJxMREOnDggJSYmCi/cnJy5DSjRo2S6tSpI+3bt086ceKE5OvrK/n6+srHi4eDd+7cWYqIiJB27NghWVlZieHgj3D/KDNJEs+5MoSGhkpaWlrSzJkzpUuXLkmrV6+W9PX1pV9//VVOExwcLJmamkqbN2+WTp8+LfXs2bPUYcteXl7S8ePHpcOHD0sNGjSo8cPB7zd48GDJ3t5eHna/YcMGydLSUpo0aZKcRjznx5OZmSmdOnVKOnXqlARI8+bNk06dOiVdvXpVkqTKea5paWmStbW1NHDgQOns2bPSmjVrJH19fTHs/nm3cOFCqU6dOpJSqZRatGghHTt2rKqL9FwBSn0tX75cTnPnzh3p3XfflczMzCR9fX2pV69eUmJiolo+cXFxUteuXSU9PT3J0tJS+uCDD6T8/PxnfDfPlwcDIvGcK8dff/0lNW7cWNLR0ZFcXV2l//3vf2rHi4qKpC+++EKytraWdHR0pI4dO0oXL15US3P79m1pwIABkqGhoWRsbCwNHTpUyszMfJa3Ua1lZGRIEyZMkOrUqSPp6upK9erVkz777DO1YdziOT+e/fv3l/o7efDgwZIkVd5zjYyMlNq2bSvp6OhI9vb2UnBwcKWUXyFJ903PKQiCIAiCUAOJPkSCIAiCINR4IiASBEEQBKHGEwGRIAiCIAg1ngiIBEEQBEGo8URAJAiCIAhCjScCIkEQBEEQajwREAmCIAiCUOOJgEgQhGolLi4OhUJBREREVRdFduHCBVq1aoWuri5Nmzat6uIIgvAUiIBIEAQ1Q4YMQaFQEBwcrLZ/06ZNKBSKKipV1Zo8eTIGBgZcvHhRbS0mQRBeHCIgEgShBF1dXWbPnk1qampVF6XS5OXlPfa5MTExtG3bFkdHRywsLCqxVIIgVBciIBIEoQR/f39sbGyYNWtWmWmmTJlSovno22+/xcnJSd4eMmQIr732Gl999RXW1taYmpoybdo0CgoKmDhxIubm5tSuXZvly5eXyP/ChQu0bt0aXV1dGjduzMGDB9WOnz17lq5du2JoaIi1tTUDBw7k1q1b8vH27dszduxYgoKCsLS0JCAgoNT7KCoqYtq0adSuXRsdHR2aNm3Kjh075OMKhYLw8HCmTZuGQqFgypQppeaTmZlJYGAgBgYG2NraMn/+fNq3b09QUJCcZtWqVfj4+GBkZISNjQ1vvvkmycnJ8vEDBw6gUCjYuXMnXl5e6Onp8fLLL5OcnMz27dtxc3PD2NiYN998k5ycHLV7mDVrFnXr1kVPTw9PT0/+/PNP+XhqaiqBgYFYWVmhp6dHgwYNSn3mglCTiYBIEIQSNDU1+eqrr1i4cCHXrl17orz27dvH9evX+eeff5g3bx6TJ0+me/fumJmZcfz4cUaNGsU777xT4joTJ07kgw8+4NSpU/j6+tKjRw9u374NQFpaGi+//DJeXl6cOHGCHTt2cOPGDfr27auWx8qVK1EqlYSEhLB48eJSy/fdd98xd+5cvvnmG06fPk1AQACvvvoqly5dAiAxMZFGjRrxwQcfkJiYyIcfflhqPu+//z4hISFs2bKF3bt3c+jQIU6ePKmWJj8/n+nTpxMZGcmmTZuIi4tjyJAhJfKaMmUK33//PUeOHCEhIYG+ffvy7bff8ttvv/H333+za9cuFi5cKKefNWsWv/zyC4sXL+bcuXO89957vPXWW3IQ+cUXX3D+/Hm2b99OVFQUP/74I5aWlg951wShBqqUJWIFQXhhDB48WOrZs6ckSZLUqlUr6e2335YkSZI2btwo3f8rY/LkyZKnp6faufPnz5ccHR3V8nJ0dJQKCwvlfS4uLtJLL70kbxcUFEgGBgbS77//LkmSJMXGxkqA2grW+fn5Uu3ataXZs2dLkiRJ06dPlzp37qx27YSEBAmQV8/28/OTvLy8Hnm/dnZ20syZM9X2NW/eXHr33XflbU9PT2ny5Mll5pGRkSFpa2tL69atk/elpaVJ+vr60oQJE8o8LywsTALk1byLVwvfs2ePnGbWrFkSIMXExMj73nnnHSkgIECSJEm6e/eupK+vLx05ckQt72HDhkkDBgyQJEmSevToIQ0dOrTMcgiCIEmihkgQhDLNnj2blStXEhUV9dh5NGrUCA2Ne79qrK2t8fDwkLc1NTWxsLBQazoC8PX1lX/W0tLCx8dHLkdkZCT79+/H0NBQfrm6ugKq/j7FvL29H1q2jIwMrl+/Tps2bdT2t2nTpkL3fOXKFfLz82nRooW8z8TEBBcXF7V04eHh9OjRgzp16mBkZISfnx8A8fHxaumaNGki/2xtbY2+vj716tVT21f8vC5fvkxOTg6dOnVSex6//PKL/CxGjx7NmjVraNq0KZMmTeLIkSPlvjdBqCm0qroAgiBUX+3atSMgIIBPPvmkRNOOhoYGkiSp7cvPzy+Rh7a2ttq2QqEodV9RUVG5y5WVlUWPHj2YPXt2iWO2trbyzwYGBuXO82nLzs4mICCAgIAAVq9ejZWVFfHx8QQEBJTo8H3/83nU88rKygLg77//xt7eXi2djo4OAF27duXq1ats27aN3bt307FjR8aMGcM333xT6fcpCM8rUUMkCMJDBQcH89dff3H06FG1/VZWViQlJakFRZU5d9CxY8fknwsKCggPD8fNzQ2AZs2ace7cOZycnHB2dlZ7VSQIMjY2xs7OjpCQELX9ISEhuLu7lzufevXqoa2tTVhYmLwvPT2d6OhoefvChQvcvn2b4OBgXnrpJVxdXUvUij0Od3d3dHR0iI+PL/EsHBwc5HRWVlYMHjyYX3/9lW+//Zb//e9/T3xtQXiRiBoiQRAeysPDg8DAQBYsWKC2v3379ty8eZM5c+bwxhtvsGPHDrZv346xsXGlXHfRokU0aNAANzc35s+fT2pqKm+//TYAY8aMYcmSJQwYMIBJkyZhbm7O5cuXWbNmDUuXLkVTU7Pc15k4cSKTJ0+mfv36NG3alOXLlxMREcHq1avLnYeRkRGDBw+WR87VqlWLyZMno6GhIc/dVKdOHZRKJQsXLmTUqFGcPXuW6dOnV+yhlHHtDz/8kPfee4+ioiLatm1Leno6ISEhGBsbM3jwYL788ku8vb1p1KgRubm5bN26VQ4uBUFQETVEgiA80rRp00o0abm5ufHDDz+waNEiPD09CQ0NLXME1uMIDg4mODgYT09PDh8+zJYtW+SRUcW1OoWFhXTu3BkPDw+CgoIwNTVV669UHuPHj+f999/ngw8+wMPDgx07drBlyxYaNGhQoXzmzZuHr68v3bt3x9/fnzZt2uDm5oauri6gqqFZsWIF69atw93dneDg4Eprspo+fTpffPEFs2bNws3NjS5duvD3339Tt25dAJRKJZ988glNmjShXbt2aGpqsmbNmkq5tiC8KBTSg50ABEEQhCeWnZ2Nvb09c+fOZdiwYVVdHEEQHkE0mQmCIFSCU6dOceHCBVq0aEF6ejrTpk0DoGfPnlVcMkEQykMERIIgCJXkm2++4eLFiyiVSry9vTl06JCYAFEQnhOiyUwQBEEQhBpPdKoWBEEQBKHGEwGRIAiCIAg1ngiIBEEQBEGo8URAJAiCIAhCjScCIkEQBEEQajwREAmCIAiCUOOJgEgQBEEQhBpPBESCIAiCINR4IiASBEEQBKHG+z/ejg/Wgw5GoQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for score_historie in elo_rankings['score_histories']:\n",
    "    plt.plot(score_historie)\n",
    "\n",
    "plt.xlabel('Number of games')\n",
    "plt.ylabel('Elo score')\n",
    "plt.title('Elo score histories')\n",
    "plt.legend(elo_rankings['model'])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ClinicalNotes",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}