CMAB - Initial Work.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "%matplotlib inline\n",
    "from matplotlib.pyplot import figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.5 , 0.91, 0.9 , 0.82, 0.85, 0.79, 0.89, 0.04, 0.66, 0.21])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s = np.round_(np.random.uniform(0.01, 1, 10), decimals=2)\n",
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>round</th>\n",
       "      <th>arm1</th>\n",
       "      <th>arm2</th>\n",
       "      <th>arm3</th>\n",
       "      <th>arm4</th>\n",
       "      <th>arm5</th>\n",
       "      <th>arm6</th>\n",
       "      <th>arm7</th>\n",
       "      <th>arm8</th>\n",
       "      <th>arm9</th>\n",
       "      <th>arm10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.209421</td>\n",
       "      <td>0.932188</td>\n",
       "      <td>0.398000</td>\n",
       "      <td>0.685312</td>\n",
       "      <td>0.726873</td>\n",
       "      <td>0.939960</td>\n",
       "      <td>0.047775</td>\n",
       "      <td>0.689492</td>\n",
       "      <td>0.036028</td>\n",
       "      <td>0.578531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.275881</td>\n",
       "      <td>0.464951</td>\n",
       "      <td>0.324978</td>\n",
       "      <td>0.773546</td>\n",
       "      <td>0.002113</td>\n",
       "      <td>0.748151</td>\n",
       "      <td>0.383163</td>\n",
       "      <td>0.683434</td>\n",
       "      <td>0.836811</td>\n",
       "      <td>0.907527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.256143</td>\n",
       "      <td>0.983893</td>\n",
       "      <td>0.215039</td>\n",
       "      <td>0.198314</td>\n",
       "      <td>0.255343</td>\n",
       "      <td>0.957247</td>\n",
       "      <td>0.384456</td>\n",
       "      <td>0.646623</td>\n",
       "      <td>0.543592</td>\n",
       "      <td>0.672722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.134960</td>\n",
       "      <td>0.682767</td>\n",
       "      <td>0.623785</td>\n",
       "      <td>0.493841</td>\n",
       "      <td>0.788148</td>\n",
       "      <td>0.860455</td>\n",
       "      <td>0.669991</td>\n",
       "      <td>0.680875</td>\n",
       "      <td>0.900057</td>\n",
       "      <td>0.603718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.652856</td>\n",
       "      <td>0.577864</td>\n",
       "      <td>0.006951</td>\n",
       "      <td>0.161430</td>\n",
       "      <td>0.395597</td>\n",
       "      <td>0.708243</td>\n",
       "      <td>0.424971</td>\n",
       "      <td>0.691223</td>\n",
       "      <td>0.731164</td>\n",
       "      <td>0.971067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   round      arm1      arm2      arm3      arm4      arm5      arm6  \\\n",
       "0      1  0.209421  0.932188  0.398000  0.685312  0.726873  0.939960   \n",
       "1      2  0.275881  0.464951  0.324978  0.773546  0.002113  0.748151   \n",
       "2      3  0.256143  0.983893  0.215039  0.198314  0.255343  0.957247   \n",
       "3      4  0.134960  0.682767  0.623785  0.493841  0.788148  0.860455   \n",
       "4      5  0.652856  0.577864  0.006951  0.161430  0.395597  0.708243   \n",
       "\n",
       "       arm7      arm8      arm9     arm10  \n",
       "0  0.047775  0.689492  0.036028  0.578531  \n",
       "1  0.383163  0.683434  0.836811  0.907527  \n",
       "2  0.384456  0.646623  0.543592  0.672722  \n",
       "3  0.669991  0.680875  0.900057  0.603718  \n",
       "4  0.424971  0.691223  0.731164  0.971067  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N=1000\n",
    "round_t = list(range(1,N+1))\n",
    "lower, upper = 0, 1\n",
    "\n",
    "a1 = scipy.stats.truncnorm.rvs((lower-0)/s[0],(upper-0)/s[0],loc=0,scale=s[0],size=N)          #mu, sigma = 0.0, ran\n",
    "a2 = scipy.stats.truncnorm.rvs((lower-0.1)/s[1],(upper-0.1)/s[1],loc=0.1,scale=s[1],size=N)    #mu, sigma = 0.1, ran\n",
    "a3 = scipy.stats.truncnorm.rvs((lower-0.2)/s[2],(upper-0.2)/s[2],loc=0.2,scale=s[2],size=N)    #mu, sigma = 0.2, ran\n",
    "a4 = scipy.stats.truncnorm.rvs((lower-0.3)/s[3],(upper-0.3)/s[3],loc=0.3,scale=s[3],size=N)    #mu, sigma = 0.3, ran\n",
    "a5 = scipy.stats.truncnorm.rvs((lower-0.4)/s[4],(upper-0.4)/s[4],loc=0.4,scale=s[4],size=N)    #mu, sigma = 0.4, ran\n",
    "a6 = scipy.stats.truncnorm.rvs((lower-0.5)/s[5],(upper-0.5)/s[5],loc=0.5,scale=s[5],size=N)    #mu, sigma = 0.5, ran\n",
    "a7 = scipy.stats.truncnorm.rvs((lower-0.6)/s[6],(upper-0.6)/s[6],loc=0.6,scale=s[6],size=N)    #mu, sigma = 0.6, ran\n",
    "a8 = scipy.stats.truncnorm.rvs((lower-0.7)/s[7],(upper-0.7)/s[7],loc=0.7,scale=s[7],size=N)    #mu, sigma = 0.7, ran\n",
    "a9 = scipy.stats.truncnorm.rvs((lower-0.8)/s[8],(upper-0.8)/s[8],loc=0.8,scale=s[8],size=N)    #mu, sigma = 0.8, ran\n",
    "a10 = scipy.stats.truncnorm.rvs((lower-0.9)/s[9],(upper-0.9)/s[9],loc=0.9,scale=s[9],size=N)   #mu, sigma = 0.9, ran\n",
    "\n",
    "df = pd.DataFrame({\"round\":round_t, \"arm1\":a1,\"arm2\":a2,\"arm3\":a3,\"arm4\":a4,\"arm5\":a5,\"arm6\":a6,\"arm7\":a7,\"arm8\":a8,\"arm9\":a9,\"arm10\":a10})\n",
    "df.head() #.head() just shows first 5 rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Distribution of Rewards for Each Arm')"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEPCAYAAACjjWTcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xcdZ3/8VfSFsrQcpm23KpQpfgxysWlgIUFBaUl7cqKeF0EAwsrsCvIFvTH+mMR/VVRoboGFywoUq8oS3EB29CqgFBEbLiUQvqxAYPV0tJ0Cm2ZXpImvz/OSZlOJ8lMMmfmJOf9fDz6SM7t+/3M6eQz3/mec77fmu7ubkREZHirrXYAIiISPSV7EZEEULIXEUkAJXsRkQRQshcRSQAlexGRBBhZ7QCkeGY2CXgBeDZcVQtsBv7L3X8R7vNloNXdf9hHOdcCz7j7/xbYtvN4M+sGJrh7ewkxHg9c6O6XmNlxwNXu/pFijx8IMxsBzAfqgEZ3/07OtvOBbwN/DlfVAPsAjwCfdvetUcbWFzPbDBzp7m1F7j8duA1YC7zX3bcMoM5TgYWA521qd/fTSy0vLLMN+Ii7Ly1i36OAZQTvi68PpD4ZGCX7oWeLu7+rZ8HMDgN+Y2Y73P1ud7+2iDLeBzxfaEORx/flncCbwrKWApEm+tBE4Axgb3ffUWD7I+7+gZ4FMxsNPAo0AHMrEF+5fAK4zd1nD7KcF3LfQxX2r8BPgM+Y2Rx376xSHImjZD/EuftLYUv9c8DdZnYHsNzdbzSzLwEfArYD64HzgbOB44AbzGwH8EEgDRwO3A8c2HN8WMVXwtZ6LXCNu98ftpY/0pNAe5aBS4EvA/ua2Q+AecB33P1IM9sX+G/gXUA3QevyC+7eaWZbga8B04GDgW+4+y35r9XMTgFuAFLha7oGWAI0AaOAZjP7sLu/0M9pGwfsC2TCcicC3wEODcu5092/ama/BO5z9++b2YnAY8Dh7v6imV0DjAW+SfCBcSBwEPAS8DF3fyVs8f4BOBr4AtAO3BS+/j+G5xQzGwP8ADgC6AKagYvdvSvntX8OOAvYEp7LL4R1vx/YEdbz7+6+Kb9ed7+nn/Oxk5kd2MfreVu47YAwztnu/vPw0IvN7Lvhth+5+/8tUPZY4JPAuwneBx8B7gy3XQecCBwCPAO0ErwnJxK8J5qBBwk+oN8CfN7df1bs6xL12Q8XzwBH5a4wszcDVwDHu/txwCLg3e7+38BS4HM5SSDl7u909/9ToOwX3f1Y4FxgnplN6C0Id18FXEvQkr4gb3MjwQfOUQQfNscAV4Xb9iToRjiJIAF8K2x9576eccD/AJ9196MJ/uh/DIwHZhJ+4+kl0Z9iZk+b2QozWwf8ArjR3e8Kt/8IuN3dpwAnAKeb2ccIuoZmhPvUA2uAnq6OfwTuJmht/97dTwTeCmSB83LqXu7udcCvgLuAK9397wgS117hPh8Cxoat7ePDdW/NO7c3APcC33L3zxF80B0SnsdjCP6Wb8ivt5dEf3h4PnL/9STnvl7PncBd7v5OgnP+VTPbJ9y2NXyfnQBcGb7/8p0H/MndWwgaAv+et/0w4O/c/dxw+eTw3Bwb1vcOd38P8BngSwXKlz4o2Q8P3QR/lLn+RvAh8KSZ3Qg87e6/7OX4R/so+7sA7r6coOvnxAHGOIOgld/t7tvCcmfkbO+5fvAkQfLfO+/4dxNcS/hDGM9zBK36U4uo+5Ewkb6DoGU9jiDxYmZ7A+8F/p+ZPQ08TtDCfxdwH3CqmY0k6CaaDUwzs0MIWrB/dPdvA4+Z2SzgZuBIYExu3eHPo4AOd/9NGP/PgE3htkeBd5rZQ8DVBNdgWvt5TTOA77p7R/gN4CZ2PZ+PFD4MCLtx8v59JYyr4OsxszTBh8r3wv1Wufvh7r4xLPOn4fo1BNcUDihQ7yUESR6CD+op4TemHo/ndev82t1fC69NrCb4BgfBdat0H69PClCyHx6O542LtgCECeC9BF036wlay9/o5fjNfZSd2wdeC3QQfLjU5Kzfo4gYa8PjcpdH5SxvCePu2Se3fIAReccXKqNP7t7l7l8G2oA7csqtAU7qSXzAVOCr7r4BeBo4k+Ci7g+BUwi6U+5x924z+zpB19U64FaCb1C5seee2/zX1BnG9WdgMnB9WM+vzezMfl5O/vnIPxd9/Z/2qo/X05OEu3P2NTPr+XbSkVNM/vujpwvuSODzYTfT7wm64nJb9/kxb8tb7kAGTMl+iAv7Uf8TmJO3/hhgOdDi7tcD3+KNLoJOik+S54flHUuQkP5AkAiONLPRZjaKXS/C9lb2AwQX5WrMbE/g08DiImOAIDm83cxOCON5J/Ae4KESyujxb8B0M/tg2DJ9HJgVlrsfwTeGD4b7zge+CvzG3TcBfyJofd8dbj+DoCX+I+AVYBpBIs63DKgxs5lhPf8I7B/+filBn/2isCvtAYKui740AZea2Sgzqw1fUynnszcFX094npoJus96ugmXEFz7KMalBH35b3b3Se4+CfgAcLaZHVqGuKUfSvZDz145/axPErRQ/8Pdf5W7k7s/Q9A3vdTMlgL/TJjQCPp+rzezhiLqe6uZPUXw9f0T7p4haO09DKwAfkdwDaDH4+Ex8/PKuZzgq/2z4T8HvlLkaya8/fOjwE1m9ixBt8EF7v6nYsvIKesF4Ou8cW3gHGBqWO4fgJ+5+0/C3X8JGG8k0gcIPsweC5e/DNxoZssIzuujBB+K+XV2EHwj6OkuOpsgmULwjWEE8LyZNRMk0MZ+XsZsgmsITwMtYUyfLfIUFOqzf9rMDujn9ZwDfMzMniHo4roo7LbpU3id52x2vaaAu/+W4EP8siLjlkGo0RDHIiLDn1r2IiIJoGQvIpIASvYiIgmgZC8ikgBK9iIiCRDbsXGam5t1m5CIyABMmTIl/wG++CZ7gClTplQ7BBGRIaW5ubngenXjiIgkgJK9iEgCKNmLiCSAkr2ISAIo2YuIJEBkyd7M3h1OxpC//kwz+6OZ/d7M/iWq+kVE5A2RJHsz+zzBkLj5U8uNIhhXfTrBxBqfNrODoohBRETeENV99i8QjF/9o7z1dQRTy20AMLNHCWb+uYsBaGpqYsGCBbutz2QyAKTThWcumzlzJvX19QOpsugY4hJHHGKISxxxiCEucVQyhriIw7kY6PuiHDFEkuzd/W4zm1Rg0z7AaznLm+hjppuWlpY+61m9ejXZbP7Uq7Bu3ToARo8evdu2nuP6K7tYvcUQlzjiEENc4ohDDHGJo5IxxEUczsVA3xfliCGyyUvCZH+nu0/NWXc08DV375ma7VvAEnf/n/zjm5ubuwf6BO3ll18OQGNjf5P9RCsOccQhhrjEEYcY4hJHHGKIi7ici3LF0dzcHIvhElqAI8KZ6jcTzCF6Y4VjEBFJnIokezM7Bxjj7rea2SyCeTxrgdvd/W+ViEFEJMkiS/bu3gZMDX//ac76+wgmKxYRkQrRQ1UiIgmgZC8ikgBK9iIiCaBkLyKSAEr2IiIJoGQvIpIASvYiIgmgZC8ikgBK9iIiCaBkLyKSAEr2IiIJoGQvIpIAlR7iWERkWGtsbKS1tbXk41auXAm8Ma59sSZPnlzUMUr2IiJl1NraylPPt7BjwoElHVezx54ALF2XKfqYEevWFr2vkr2ISJntmHAgr3/4vMjr2fvu/Gm+e6c+exGRBFCyFxFJACV7EZEEULIXEUkAJXsRkQRQshcRSQAlexGpmvb2di677DLWr19f7VCGPSV7EamauXPn8swzzzB37txqhzLsKdmLSFW0t7ezePFiABYtWqTWfcSU7EWkKubOnUtXVxcAXV1dat1HTMleRKri17/+9S7LPa18iYaSvYhIAijZi0hVHHLIIX0uS3kp2YtIVaxbt67PZSkvJXsRqYqDDjqoz2UpLyV7EamKl19+uc9lKa9IJi8xs1rgZuAYYBtwkbu35mz/JHAlsAO43d1viSIOEam+pqYmFixYsNv6jo6O3ZZzp9ebOXMm9fX1kcdXbplMhhHr1pY0schAjVi3lsyI4vaNqmV/FjDa3U8Ergbm5G2/ETgd+HvgSjPbP6I4RCSmeu6x721ZyiuqaQlPBpoA3P1xMzsub/syYF+gE6gBuiOKQ0SqrL6+vmAL/VOf+hRtbW07lydNmkRjY2MFI4tGOp3mxR1UbFrCdDpd1L5RJft9gNdylneY2Uh37wyXlwPNwOvAfHd/tVAhLS0tA6o8m80O6vhyiUMccYghLnHEIYa4xBGHGM4991xmz569c/m8884bdDw///nP+etf/1rSMatWrQLgoosuKrm+N73pTXz84x/fZV3Pua2UbDZb1HmLKtlvBMbmLNf2JHozOxr4B+AtwGbgx2b2UXe/K7+Qurq6AVWeSqUGdXy5xCGOOMQQlzjiEENc4ohDDHV1ddxwww1s27aNSZMmMW3atEGXmclkaGtzxk8ovrNgz9HBz82vryiprvZ1NaRSqd3OYSqVgte3llTWYOTH0NzcXHC/qJL9EuBM4BdmNhV4Nmfba8AWYIu77zCzVwD12Ysk0GGHHUZrayvXXntt2cocP6Gbs8/eXrbyejN//h6R11FOUSX7e4BpZvYYQZ/8BWZ2DjDG3W81s7nAo2a2HXgBuCOiOEQkxlKpFEcffTSTJ0+udijDXiTJ3t27gEvyVq/I2f5d4LtR1C0iIrvTQ1UiIgmgZC8ikgBK9iIiCaBkLyKSAFHdjSMiFdLY2Ehra2v/O+ZYuXIlwC5j0RRr8uTJAzpOqkvJXmSIa21txZe38OaxxQ8RPLY7eJIo+9KGkupatWlNSftLfCjZiwwDbx57EFeecEHk9cx54geR1yHRUJ+9iEgCqGUvMkAD6SuHgfeXq69cBkPJXoakOFyUbG1tZfkzzzB2j9L+jLp3BOO2v9TyXNHHbNre2f9OVaQPvvhTspchqbW1leeebWG/1AFFH1O7Y08A/vbC+pLqejX7Sq/bxu4xkhMOjH4cvyfWlnYhtdJaW1tZvnw5Y8aMKem47u5gdMrcce37s3nz5pLqkICSvQxZ+6UO4LS3fyLyeh5ccWfkdQwHY8aM4dhjj428nieffDLyOoYjJXspib6uiwxNSvZFqmQfcW8JLi791Cuefpri7+gO7BX+fPXpp4s+Rnd0i5SPkn2RWltb+dPyJzl0zI6ij9mnuwaArW1/LPqYv2zufar41tZWnnruKdiv6OJ23lz71N+eKuEgoOBEkYGDgAupKa28Afi+piYWKRsl+xIcOmYH1xwX7cWh2Uv7ucC1H3Sd2hVpDAC1D+kRDCleJpNh06ZNFelP37RpE5lMJvJ6hhslexGRMhuxbi173/2jko6pyQYNye5U8Xc0jVi3Fiaki9pXyV5EBi2dTrNx48aK3Y2TTheX4KphoFMsrlwZ3BJ8xGGHFn/QhHTR9SnZi4iU0UDvHus5rrGxsZzh7KSOWRGRBFDLXkTKYvPmzSVfoN2+fTsAe+yxR0n1SOmU7EVk0AbeTx08BzJp0qSK1JdkSvYiMmhx7aeWN6jPXkQkAZTsRUQSQMleRCQBlOxFRBJAyV5EJAGU7EVEEkDJXkQkAZTsRUQSQMleRCQBInmC1sxqgZuBY4BtwEXu3pqz/Xjgm0ANwexz57r71ihiERGR6Fr2ZwGj3f1E4GpgTs8GM6sBbgMucPeTgSbgsIjiEBERokv2PUkcd38cOC5n29uA9cAVZvYwkHZ3jygOEYmxbDbLsmXLaG1t7X9nGZSoBkLbB3gtZ3mHmY10905gPHAScBmwErjfzJrd/Tf5hbS0tAyo8mw2O6jjeyuzUhc4stlswdh7XlelFIojDjHEJY44xBCnOAaira2Nrq4urrnmGr74xS8OuryhfC6iyFu5okr2G4GxOcu1YaKHoFXf6u7PA5hZEzAF2C3Z19XVDajyVCo1qON7K7NSFxVSqVTB2FOpFGyoUBC9xJFKpdheuRD6PBcb2FLVOHreZ9WMoWd9lm1Vj6M3TU1NLFiwYLf12Wx253j2q1evZs6cObuc05kzZ1JfX19ybJtfL+mQQSn1XPRXFgw+bzU3NxdcH1VjdQkwE8DMpgLP5mx7ERhjZj0DUp8CPBdRHCISUy+99FKfy1JeUbXs7wGmmdljBHfcXGBm5wBj3P1WM7sQ+Gl4sfYxd/9VRHGISJXV19cXbKG/5z3v2WV527ZtGtc+QpEke3fvAi7JW70iZ/tvgROiqFtEhoZJkybR1ta2y7JERzNViUhVfOYzn+Gqq67aufzZz3520GVmMhna19Uwf37xc9oOVPu6GkaOyEReT7noCVoRqYpHHnlkl+WHH364SpEkg1r2IlIVixcv3mV50aJFzJo1a1BlptNpOne0cfbZ0d8zNn/+HqTT6cjrKRe17EWkKqZNm8aoUaMAGDVqFNOnT69yRMNb7Fv2jY2NJT9dt3LlSmBgM95Pnjx5QMeJSGkaGhpYuHAhALW1tTQ0NFQ5ouGt32RvZiOA84FDgQeB5e7eHnFcO7W2tvLUs8/TlSr+61LNjuBlNb+wpqS6arND52KLyFA3fvx4ZsyYwb333suMGTMYN25ctUMa1opp2c8FVgPTgKXADwkfmKqUrlSare/4QOT1jH7+/sjrEJE3NDQ00NbWplZ9BRTTZ3+4u18LbHH3+4B9I45JRBJi/Pjx3HTTTWrVV0AxLfuRZjYewMzGAl3RhiQipchkMqzbtJY5T/wg8rpWbVrDhEx35PVI+RWT7K8hGOvmYOBx4IpIIxIRkbLrN9m7+8OAmdkEoN3d9bEuEiPpdJrRm2q48oQLIq9rzhM/IJXeP/J6pPyKuRvnQaA7Zxl3f1+kUYmISFkV043TM6BZDcG488dEF46IiEShmG6c3CkDV5jZP0cYj4iIRKCYbpxP5ywezK4zUImIyBBQTDfOwTm/bwU+FlEsIkNKJpNh0/ZOnlgb/VyRm7Z3ksnoCW8ZuF6TvZm9Lfz1Z3mboh8oWkREyqqvlv3cXtZ3A7obRxIvnU6zae3LnHBg9LciPrF2w5AaTlfip9dk7+6nFVpvZmrZS9VlMhlezb7CgyvujLyuV7OvsFemJvJ6RKJUzAXai4FZwCiC2y87gLf1eZCIiMRKMRdo/wU4lWDYhLvQcAkSA+l0mi0bujnt7Z+IvK4HV9ypLhQZ8opJ9uvd/WUzG+vuD5nZlyOPSgrKZDLwKtQ+VIEJxl6FzF66+0NkuCgm2b9qZmcB3WGXzoSIY5IYy2QyrAW+T/RDJL0MdOl2Q5GyKCbZTwCOBK4GrgIujTSimAqGkR3B7KVjIq3npU0jmNBLgkun07y05SW6To1+lOnah2rVdSEyjBST7D9PMC3hQuCXwItRBiTxlk6nqf3LX7iQ6O9O+T7d7KcPHJGyKGZsnKXAUjPbH7gFWAnsGXVgcZNOp0ltfIFrjtscaT2zl45htBKciJRZMbdenkLQsj+e4G6cqyKOSUREyqyYbpwrgNuAizRxiYjI0FRMN86HKxGIiIhEpwI3bIuISLUp2YuIJEAxffYlM7Na4GaCKQy3EfT3txbY71Yg4+5XRxGHiIgEomrZnwWMdvcTCR7GmpO/Q/g07lER1S8iIjmiSvYnA00A7v44cFzuRjM7EZhK72Pmi4hIGUXSjQPsA7yWs7zDzEa6e6eZHQxcB3yIfqY4bGlpIZvNRhRiYdlslpaWloLrK3WBo68YKqlQHHGIIS5xxCGGOMURB0P5XPTEHtW5jSrZb2TXiclr3b0z/P2jwHhgAXAQkDKzFe5+R34hdXV1pFKpsLjKSKVS1NXVFVy/NQYxEP10p33GkUql2F65EPo8FxvYUtU4gvdm5fR1LrJsq3occZBKpdj8emXrK9e56Hk/Dba85ubmguujSvZLgDOBX5jZVODZng3u3gg0ApjZ+cDbCyV6EREpn6iS/T3ANDN7jGB2qwvM7BxgjLvfGlGdIiLSi0iSvbt3AZfkrV5RYL87oqhfRGSo6ejooK2tjfXr1zNu3Liyl6+HqkREYmD16tW8/vrrzJ0bzU2KUXXjiIhInqamJhYsWLDb+o6ODjZs2LBzn1WrVjFq1Kid22fOnEl9ff2g6lbLXkSkylavXt3ncjmoZS8iUiH19fUFW+innXbaLssbN26ksbGxrHWrZS8iUmVdXV19LpeDkr2ISJXV1tb2uVwO6sYRkWGlfV0N8+fvUfT+PSMslPpAdPu6Gg6YUNoxvTn99NN54IEHdi5PmzatPAXnULIXkWFj8uTJJR+zcuVKAA447IiSjjtgwsDqK+Tiiy/eJdlffPHFZSk3l5K9iAwbl19++YCPKfcF0VLV1NTQ3d1NTU1NJOWrz15EpMrmzZvX53I5KNmLiFTZokWL6O7uBqC7u3uXLp1yUbIXEamyAw88sM/lcoh9n30mk6E2u57Rz98feV212fVkMsVfxRcRKYe1a9f2uVwOatmLiFTZ9OnTd16Yramp4Ywzzih7HbFv2afTaf68YTtb3/GByOsa/fz9pNPpyOsREcnV0NDAvffeu8tyuallLyISA7kt+ygo2YuIVNm8efN2DpFQW1urWy9FRIajxYsX09nZCUBnZyeLFi0qex2x77MXkf6t2rSGOU/8oOj9N27bDMA+e44puR5j/5KOkf5NmzaNBQsW0NHRwahRo5g+fXrZ61CyFxniBjI+y99WtgNw0GFvLuk4Y/+yjQcjb2hoaGDhwoVA0I0TxQVaJXuRIW4ojwcjgfHjxzNjxgzuvfdeZsyYEcmE40r2IoOwaXsnT6zdUNIx23YEE1PsOaL4S2abtneWVIcMPQ0NDbS1tUXSqgcle5EBG2h3Rs+QuocdUdqQuuo+Gd7Gjx/PTTfdFFn5SvYiAzSQ7pPc49SFIpWkWy9FRBJALXsZsl7NvsKDK+4sev+tHa8DMHrU3iXXM5HyXzATydXe3s6XvvQlrrvuOl2gFekxsOnnMgBMPPzQko6byDj1l0vk5s2bx7Jly5g3bx6zZs0qe/lK9lKyNcD36S7pmM3hz1Ie4VkD7NfLNt1uKMNJe3s7CxcupLu7m4ULF9LQ0FD21r2SvZRkoC3cdeEdKG8q4Q6U/QZRn8hQMm/evJ0zVXV1dUXSuleyH2pehdqHSriuvjX8Obr0epi4+2rdgSJSfosXL6ajowOAjo4OFi1apGRfTX/ZPILZS4vviHhtezBU6b57FN/l8ZfNI3hbL9sG1k8dtKiPmFjaPd1MVKtapFI0Nk6MDCTxrQoT7YGTik+0b+ujLvVTiwxPDQ0NLFiwAAjGsx8yY+OYWS1wM3AMsA24yN1bc7b/E3AFsANYBvyru3dFEUu5KNGKSFTGjx/PxIkTaWtr45BDDonk1suoHqo6Cxjt7icCVwNzejaY2V7AbOA0dz8J2BeIfs5BEZGYam9vZ/Xq1QCsXr2a9evXl72OqJL9yUATgLs/DhyXs20bcJK7Z8PlkbxxGVFEJHFy78bp7u6OZKaqqPrs9wFey1neYWYj3b0z7K5ZC2BmlxHcer24UCEtLS1ks9lCmyKTzWZpaWkpW1lA2cobqjHEJY44xBCXOOIQQ1zE4Vw88MADu9yN09TUxIwZM8paR1TJfiMwNme51t13jtEa9ul/g+B65IfdveDtKnV1daRSqbC4ykilUtTV1ZWtLKBs5Q3VGOISRxxiiEsccYghLuJwLs4444xd7sapr68fcDzNzc0F10fVjbMEmAlgZlOBZ/O2zyW48/usnO4cEZFEamhooKYmuFV7qM1UdQ8wzcweA2qAC8zsHIIum6XAhcAjwG/NDODb7n5PRLGIiMTakJ2pKuyXvyRv9Yqc3zW0sohIDs1UJSKSAFHPVKUWtohIAijZi4gkwJDoxqnNZhj9/P1F71/TsQWA7lF7lVwPHFTSMSIiQ0Hsk/2gRno8vNTEfZBGehSRYSn2yV4DkImIDJ767EVEEkDJXkQkAZTsRUQSQMleRCQBlOxFRBJAyV5EJAGU7EVEEkDJXkQkAZTsRUQSQMleRCQBlOxFRBJAyV5EJAGU7EVEEkDJXkQkAZTsRUQSQMleRCQBlOxFRBJAyV5EJAGU7EVEEkDJXkQkAZTsRUQSQMleRCQBlOxFRBJAyV5EJAGU7EVEEmBkFIWaWS1wM3AMsA24yN1bc7afCVwLdAK3u/ttUcQhIiKBSJI9cBYw2t1PNLOpwBzggwBmNgr4FnA88DqwxMzuc/c1EcUiIkJTUxMLFizYbf3KlSsBuPzyywseN3PmTOrr6yONrRKiSvYnA00A7v64mR2Xs60OaHX3DQBm9ihwCnBXqZXE4T+vtxjiEkccYohLHHGIIS5xJCXBFWPcuHHVDqEiokr2+wCv5SzvMLOR7t5ZYNsmYN9yVh6X/7w4xBGHGCAeccQhBohHHHGIodLq6+sT8wFWSE13d3fZCzWzbwKPu/svwuW/uvubwt+PBr7m7jPD5W8BS9z9f3LLaG5u7k6lUmWPTURkOMtms0yZMqUmf31ULfslwJnAL8I++2dztrUAR5hZGtgMvAe4sVAhdXV1EYUnIjI8NTc3F1wfVbK/B5hmZo8BNcAFZnYOMMbdbzWzWcADBLd+3u7uf4soDhERIaJk7+5dwCV5q1fkbL8PuC+KukVEZHd6qEpEJAGU7EVEEkDJXkQkAZTsRUQSQMleRCQBorr1six6u19URERKE8kTtCIiEi/qxhERSQAlexGRBIh1n31UzCwFLAYudPcV/e0fUQz/BFwB7ACWAf8aPnlcyRg+DFwNdAO3uvv3Kll/gXhuBTLufnUV6p4FXAisC1dd7O5ehTiOB75JMMzIGuBcd99awfoPAu7MWfUu4Gp3/26lYsiJ5ZPAlQR/I7e7+y1ViOE84HMEI/Xe4e7fr3D9u+Sq/iaG6kviWvbh2Pq/Aw6vYgx7AbOB09z9JIIhnj9Q4RhGAF8DTgdOBD5nZuMrGUNePBcDR1WrfuBY4FPufmr4rxqJvga4DbjA3XvmhDiskjG4+5qecwD8B/BkGFM13Ejw/vx74Eoz27+SlYd/D7OBU4H3Ap80s0kVrL9Qrto5MRRBQ21OseUNqZa9me0DfA/YDxhP8Cb8OEFrbH/gZ8BMYC/gYODbBDNkHQlc5e7/C+wJfAj4UbXiIBgX6CR3z/yrzhYAAAS1SURBVIZFjgRKar2V41yYWZ27d5rZAQQtyc2VPhdhHCcCU4G5wNurEQMwBfiPsGX7K3e/vtJxEIwftR64wsyOCuMo6UOnTOei54PnJuCT7r6j0ucijGMZQUOok+D9WdLdJGX4/3gZeNrdM2F5fyR4n7ZV6PUXylV9TQzVp6HWsp8M3Onu0wlawrPC9T9199MJvu6NDcfK/zpwKXA28GngAgB3X+Luq6oZh7t3uftaADO7DBhD8FWtYjEAhIn+bOAZghZER4kxDDoOMzsYuA74twHUXZYYwn3vJBi8733AyWY2kG9ag41jPHASwdf004H3m9n7KxxDjzOB5wbxDacccSwHmoHngPvd/dUKx7ASeKeZHRh2p7wf2LtSr7+XXFVwYqhiAhpqyX4NcJaZ/Ri4BhgVrs99Qz4V/nwVaHH3bmADMDpOcZhZrZndCEwDPhxur2gMAO4+H5gI7AF8qsQYyhHHRwmS3AKCr6XnmNn5lYwhbMX+l7u3u/t24FfA35UYw6DjIGjVt7r78+7eQdCCm1LhGHqcC9xaYt1liyOc5OgfgLcAk4ADzOyjlYwhnDr134G7gdsJurTaK1F3H+VuBMbmLNeGMwD2a6gl+6uA37v7uQRz1vbMxpJ7YbMSDw6UI465BP+pZ+V051QsBjPbx8weNrM9wwvDr+cdW5E43L3R3aeEfcRfI2j53FHJGAhaS8vNbEyY+N9H0KIs1WDjeBEYY2aTw+VTCFq1lYyhxxTgsRLrLmccrwFbgC1hN9IrBN0fFYshbDFPJZhg6VMEXYxLKlF3H5YQdP9QYGKoPg2pPnuCvu5bwqv06wn68vYcanGY2bEEd348AvzWzAC+7e73VCoGd99oZj8BfmdmHQT9oz8uof6yxFEmgz0Xr5nZF4AHCe5w+I27F54tPNo4tpvZhcBPww+dx9z9V5WMAcDMJgCbBvBts2xxuPtLZjYXeNTMtgMvAHdUOIbOsO5mgmtqc9y92JZ9VH8Xu00MVeyBeoJWRCQBhlo3joiIDICSvYhIAijZi4gkgJK9iEgCKNmLiCSAkr1IP8ys3sw+3cf268zskgLr10QbmUjxhtp99iIV5+5N1Y5BZLCU7CVRzGw+wQNsD4fDCX+DYHCqnQNWufstZvYQuw5adYS7X21m1wPHETyy3uLuPQ+1fMjMPgakgMvd/YmcOo8CGgkeglkP/LO7545vIhI5deNI0twGNIS/n0/w1GyhAatg10GrekYy3ODu0wgGLJtqZhPDff/s7u8jeDI6f+z324B/C4eEWAB8vtwvSqQ/atlL0jwA3GBmaYKxZ2YA14ejf27kjQGrYNdBqyAYq+UAM/sZwXDQY3L2/x2Auz8XDpOcqw64ORwWYxTwp/K9HJHiqGUviRIO+nYXcAvwS4KZkAoNWAW7Dww3A3izu/8T8AWCsch79j8BdnbZ/CW/WsKJUQha9aWOdyMyaGrZSxLdTjDC5BEEQ+juMmCVmfU2YNUTwH+a2eMEA6a9CBwSbnuLmf2WYLCri/OOuxT4YTg7GARdPSIVpYHQREQSQN04IiIJoGQvIpIASvYiIgmgZC8ikgBK9iIiCaBkLyKSAEr2IiIJoGQvIpIA/x+k9yUUgw0JWAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(df[['arm1','arm2','arm3','arm4','arm5','arm6','arm7','arm8','arm9','arm10']]))\n",
    "plt.title(\"Distribution of Rewards for Each Arm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "#implementing UCB, 1000 runs\n",
    "#CMAB for 2 arms being selected\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding ucb values. re-initialize to 0 at start of each round\n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt((3/2)*math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "        \n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    N[arm1_selected] += 1\n",
    "    N[arm2_selected] += 1\n",
    "    reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df.values[t, arm1_selected+1]\n",
    "    rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD2CAYAAAAtW8c3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deYDN9f7H8eeMjJ2ikBZa9DGK1Eiy36490X5V195YfoSuyE5Fk0KI0DAh1e2WLE2UfSeZEHX6ZMu1ZjfGmP38/pjj3rmWmDPLd845r8df53y+3+983++Z8fKd7/d7Pt8gt9uNiIj4l2CnCxARkeyncBcR8UMKdxERP6RwFxHxQwp3ERE/pHAXEfFD113LSsaYh4FR1toGGcZeAF621j7ieR8OdAFSgBHW2mhjTCFgNlAaOAu0s9Yey94WRETkYlc9cjfG9AOmAQUzjFUDOgFBnvdlgZ5AbaAJEGGMKQB0A7Zba+sCs4DB2d2AiIhc6lqO3HcDTwEfAxhjSgFvA72BSM86NYB11tpEINEYswuoCtQB3vGsswgYcrkdxMTE6JNUIiJeCAsLC7rc+FXD3Vo7xxhTAcAYkw+YDrwCnM+wWnHgTIb3Z4ESF41fGLuswoULX62Uy0pISKBgwYJXX9GPqOfAoJ4DQ1Z6jo+Pv+KyazrnnkEYUBGYTPppmsrGmHHAcqBYhvWKAaeB2AzjF8YuKzQ0NJOlpHO5XF5v66vUc2BQz4EhKz3HxMRccVmmwt1auwm4F8BzNP9Pa21vzzn3kcaYgkABIBTYAawDmgObgGbAGi/qFxGRTMqWWyGttUeACaSH93JgkLU2gfQj/HuNMWuBzsDr2bE/ERH5c9d05G6t/R2o+Wdj1tpI/nuB9cJYPPBsVosUEZHM0YeYRET8kMJdRMQPKdxFRPyQwl1ExAGLdi7CTDQs+veiHPn6CncRkVwUnxxPj4U9aP5pc/IH56dqqao5sh+Fu4hILok5FEPYh2FM+mESr9R8hc2dN3NLkVtyZF+Z/YSqiIhkUlxSHG+seoP3Nr5HmSJlWNJmCQ3vbJij+1S4i4jkoLX/Xku7ee3Ye2ov7aq1Y0zjMZQsVDLH96twFxHJAYkpiQxdMZR3179LhesrsKr9KuqWr5tr+1e4i4hks21HttFmbhu2H91O5wc7M7rxaIoVKHb1DbORwl1EJJukpKXw7rp3GbZyGKUKl+KbF76hecXmjtSicBcRyQa7Tu6i7dy2bDiwgWcrP8vkxyZTqnApx+pRuIuIZIHb7WZqzFT6LO5DSL4QPnnqE56/73mCgi77gKRco3AXEfHSwdiDdFrQie92f0fjuxoT1TKKW4rnzH3rmaVwFxHxwj93/JP/++b/SEhJYFLzSXSr3s3xo/WMFO4iIplwIv4E3Rd25/OfP6fmrTWZ9cQsKpaq6HRZl1C4i4hco0U7F9FpQSeOxx/nrUffom/tvlwXnDdjNG9WJSKSh5xJOEOfxX2YvmU695W+j4UvLqRa2WpOl/WnFO4iIn9i5e8raTO3DYfOHuK12q8xvMFwCl5X0OmyrkrhLiJyGcmpyQxfOZyItRHcXfJuNnTaQI1bajhd1jVTuIuIXGTPqT28MOcFvj/4PR2rdWR8s/EUDSnqdFmZonAXEfFIc6fxYcyH9FvSj+CgYD5/5nOeu/c5p8vyyjWFuzHmYWCUtbaBMaYa8D6QCiQCba21fxhjwoEuQAowwlobbYwpBMwGSgNngXbW2mM50YiISFYcPnuYNnPbsGzvMh6941GiWkZR/vryTpfltas+ickY0w+YBly4gjAeeNla2wD4CnjNGFMW6AnUBpoAEcaYAkA3YLu1ti4wCxic7R2IiGTRop2LuH/K/azfv56pLaaytM1Snw52uLbH7O0GnsrwvrW1dqvn9XVAAlADWGetTbTWngF2AVWBOsC3nnUXATn76BERkUw4k3CG7t90p/mnzSlbtCybO2+mc1jnPPVJU29d9bSMtXaOMaZChveHAYwxtYAeQD3Sj9bPZNjsLFACKJ5h/MLYZblcrkyWni4hIcHrbX2Veg4M6jlnLTmwhBE/juBE4gnaVGzDK1VeIeh4EK7jufs9z6mevbqgaoz5GzAIeMxae8wYEwtknIm+GHAayDh+YeyyQkNDvSkFl8vl9ba+Sj0HBvWcM06dP0X3hd35bMdnVCtbjYWPL6R6ueo5us8/k5WeY2Jirrgs0+FujPk76RdOG1hrT3qGNwEjjTEFgQJAKLADWAc09yxvBqzJ7P5ERLLLt7u+pdOCThw9d5Q3//Im/ev0z7PTB2RVproyxuQDJgD/Br4yxgCsstYOM8ZMID28g4FB1toEY8xkYKYxZi2QBLyQrdWLiFyDuKQ4+nzXhw9//JDKN1VmQesFhJULc7qsHHVN4W6t/R2o6Xl72cd2W2sjgciLxuKBZ7NQn4hIlizbs4wu0V3Yc2oPrz7yKm8++qZPTB+QVf7594iIBLwjcUcYsGwAM7bO4M4b7mRl+5XUK1/P6bJyjcJdRPyK2+1m2o/T+MfifxCfHM/AOgMZUn9IQBytZ6RwFxG/cfTcUcK/DmeBXcBfKvyFSc0nEXpTYN1xdIHCXUT8QvRv0XRa0IkzCWcY23gsvWr2IjjoWj6n6Z8U7iLi0+KT4+nzXR+mxEyhapmqLGu7jPtK3+d0WY5TuIuIz9pyeAsvfPUCvx7/lVcfeZURj46gwHUFnC4rT1C4i4jPSXOnMXbDWAYuG8hNRW5iSZslNLxTU1dlpHAXEZ9y7Nwx2s1rx6Jdi3gq9CkiH4+kZKHLfvwmoCncRcRnzPt1Hp2/7kxsYiwfNP+ArtW7+sUMjjlB4S4ied7phNP0+rYXs7bN4oGyDzDryVm6aHoVCncRydMW715Mx/kdORJ3hKH1hjKo3iBC8oU4XVaep3AXkTwpNjGWPt/1YdqWaVS6sRJz/zaXh255yOmyfIbCXUTynBV7V9Bhfgf2x+6nX61+vP6X1wNu+oCsUriLSJ5xPuU8PRf15P1N73N3ybtZ02ENtW6r5XRZPknhLiJ5wrI9y+jwXQf2n9vPyzVeJuKvERQJKeJ0WT5L4S4ijjp89jADlg1g5raZlC9anhXtVtCgQgOny/J5CncRcYTb7Wbmtpn0WdyH2MRY+tXqR+uyrXmgwgNOl+YXFO4ikut2ndxFl+guLN+7nDq31yHy8Ugq3VgJl8vldGl+Q+EuIrkmOTWZ0etH88bqNwjJF8KUx6YQHhYe0FPz5hSFu4jkiu8PfE/41+FsP7qdZyo/w/im4ylXrJzTZfkthbuI5Kik1CSGrRjGqHWjKFesHPNbz6elael0WX7vmsLdGPMwMMpa28AYczcwA3ADO4Du1to0Y0w40AVIAUZYa6ONMYWA2UBp4CzQzlp7LAf6EJE8KOZQDB3md2D70e289MBLjGkyhuIFijtdVkC46okuY0w/YBpw4eNhY4HB1tq6QBDQyhhTFugJ1AaaABHGmAJAN2C7Z91ZwODsb0FE8prElEQGLRvEw9Me5sT5EyxovYDIlpEK9lx0LUfuu4GngI8978OAVZ7Xi4DGQCqwzlqbCCQaY3YBVYE6wDsZ1h2STXWLSB61et9qun3TjV+O/UKHah0Y22Qs1xe83umyAs5Vw91aO8cYUyHDUJC11u15fRYoARQHzmRY53LjF8Yuy9tboBISEgLu9in1HBh8reeTCScZ/dNo5v0+j3KFyzGl7hTq3VyPw3sPc5jD1/Q1fK3n7JBTPXtzQTUtw+tiwGkg1vP6z8YvjF1WaGioF6Wk/6fg7ba+Sj0HBl/pOc2dxrQfp9F/aX/OJp2lf+3+DK432KupA3yl5+yUlZ5jYmKuuMybm0u3GGMaeF43A9YAm4C6xpiCxpgSQCjpF1vXAc0vWldE/MTWI1upHVWbLtFdqFqmKtu6biOioeaEyQu8OXLvA0QaY0IAF/CltTbVGDOB9PAOBgZZaxOMMZOBmcaYtUAS8EJ2FS4izjmffJ5hK4cxZsMYShUqxcwnZtKmahs98i4PuaZwt9b+DtT0vP4NqH+ZdSKByIvG4oFns1yliOQZG/ZvoMP8DtgTlk4PdOLdRu9yQ6EbnC5LLqIPMYnINYlPjuetNW8RsTaCW4vfyuK/L6bRXY2cLkuuQOEuIle1ePdi2s5tyx/n/qBDtQ6MazpO96zncZqtR0SuKDk1mQFLB9BkdhNuLHwjq9uvJqpVlILdB+jIXUQu699n/s3zc55n/f71hD8Yzrim4yicv7DTZck1UriLyCXm/DKH8K/DSUlL4bOnP6P1fa2dLkkySeEuIv8RlxRHz0U9+WjrR1QvV53Pnv6Mu0ve7XRZ4gWFu4jgdrv5ZPsnDFo+iP1n9jO47mCG1h9K/nz5nS5NvKRwFwlwB2MP8vKil5n761zCbg5j1hOzqF/hko+yiI9RuIsEqDR3GlM3T6Xf0n4kpiQyutFoXnnkFT3yzk8o3EUC0LYj2+j1bS9W7VtFwzsbMuWxKdxV8i6ny5JspHAXCSBp7jTe2/AeA5cPpGhIUSIfj6TTA500J4wfUriLBIgT8SdoP7890b9F08q0YnrL6ZQqXMrpsiSHKNxFAsDi3YtpP689x+OPM7HZRP7vof/T0bqfU7iL+LG4pDgGLhvI+5vep/JNlVn44kKqla3mdFmSCxTuIn5q+d7ldFrQiX2n99GzRk/ebvg2hfIXcrosySUKdxE/c/L8SYatGMbEHyZyT6l7WN1hNXVur+N0WZLLFO4ifsLtdjNj6wxe+e4VYhNj6fFQD0Y1GqXJvgKUwl3ED/wR9wedozuzwC6gXvl6TGo+iftK3+d0WeIghbuIj5vrmkvn6M6cTTzLmMZj6F2ztz5lKgp3EV91OuE0vb7txaxts3jw5gf5+MmPqXxTZafLkjxC4S7ig5btWUaH+R04dPYQQ+sNZXC9wZrBUf6Hwl3Eh8QnxzNg6QAmbJqAKWVY32k9NW6p4XRZkgd5Fe7GmPzATKACkAqEAynADMAN7AC6W2vTjDHhQBfP8hHW2uisly0SeDYd3ETbuW2xJyw9a/QkomGE7oSRK/L2qktz4DprbS3gDWAkMBYYbK2tCwQBrYwxZYGeQG2gCRBhjCmQ9bJFAkdyajLDVgyj1vRaxCfHs7TNUsY3G69glz/l7WmZ34DrjDHBQHEgGagJrPIsXwQ0Jv2ofp21NhFINMbsAqoCP2SpapEA8cuxX2gztw0/Hv6Rtve3ZULTCZQoWMLpssQHeBvucaSfkvkVuBFoAdSz1ro9y88CJUgP/jMZtrswfgmXy+VVIQkJCV5v66vUs/9Lc6cR9UsUE10TKZK/CBNqTaDhrQ05tPcQhzjkdHk5JtB+zpBzPXsb7q8A31lrBxhjbgOWAyEZlhcDTgOxntcXj18iNDTUq0JcLpfX2/oq9ezffj/9O+3ntWfVvlW0Mq2Y2mIqZYqWcbqsXBFIP+cLstJzTEzMFZd5e879FP89Ij8J5Ae2GGMaeMaaAWuATUBdY0xBY0wJIJT0i60ichG3203UliiqTq7Kj4d/ZORDI5n7t7kBE+ySvbw9cn8PiDLGrCH9iH0gsBmINMaEAC7gS2ttqjFmAulBHwwMstYmZEPdIn7lj7g/CP86nK9/+5oGFRowo9UM4g/Ha8518ZpX4W6tjQOeu8yiSx6Zbq2NBCK92Y9IILgwfUBcUhzvNXmPng/3JDgoGNfhwDr3LNlLH2IScUhcUhy9FvUiamsUYTeH8fGTHxN6U2Cdb5aco3AXccCafWsI/zqc3078xsA6AxneYLimD5BspXAXyUXnk8/Td0lfJv0widtL3M7Stkt59I5HnS5L/JDCXSQXpKalMmPrDF5f9Tr7Y/fT++HejHh0BEVCijhdmvgphbtIDjt67igvfvUiS/cspcYtNfj4yY+pX+GSew9EspXCXSSHuN1u5rjm0O2bbsQmxhL5eCSdHuik2xslVyjcRXLArpO76BLdheV7l1O9XHVmtJrBvaXvdbosCSAKd5Fs5Ha7mRozlT6L+5A/OD8Tmk6ga/WuuhNGcp3CXSSbHIw9SKcFnfhu93c0urMRUa2iuLX4rU6XJQFK4S6SRWnuNKb/OJ2+S/qSlJrEpOaT6Fa9m86ti6MU7iJZsPPETjpHd2bl7ytpUKEBH7b4kIqlKjpdlojCXcQbKWkpjFk/huGrhlMgXwHdCSN5jsJdJJO2HN5CpwWd2HJkC09WepKJzSdSrlg5p8sS+R8Kd5FrlJiSyPCVw3l3/bvcVOQmvnz2S56u/LTTZYlclsJd5Bqs37+e7gu7s/XIVjpW68joxqO5odANTpclckUKd5E/kZiSSP+l/Rn3/TjKFi3L/NbzaWlaOl2WyFUp3EWu4IeDP9Bhfgd+PvYzPR7qwdsN39ZEX+IzFO4iF0lISWDcxnEMWTGEm4veTPTz0Tx2z2NOlyWSKQp3kQy++PkLen7bkyNxR3im8jN82OJDnVsXn6RwFwH2nd7HiNUjmLZlGg+Ve4jZT87m0Tse1X3r4rMU7hLQElMSGb1+NG+ufpOk1CT61urLyEdHaqIv8Xleh7sxZgDQEggBPgBWATMAN7AD6G6tTTPGhANdgBRghLU2OqtFi2SHC88xtScsT4c+zejGo6lwfQWnyxLJFsHebGSMaQDUAmoD9YHbgLHAYGttXSAIaGWMKQv09KzXBIgwxhTIhrpFvJaSlsKgZYOoN6MeCSkJfPvit3z53JcKdvEr3h65NwG2A3OB4kBfIJz0o3eARUBjIBVYZ61NBBKNMbuAqsAPWSlaxFsHYw/Sdl5blu9dTqcHOjG+6Xjd3ih+ydtwvxEoD7QA7gAWAMHWWrdn+VmgBOnBfybDdhfGL+FyubwqJCEhwettfZV69s7SA0sZsnkISalJjHhoBE/d8RT/3v3vbKow++nnHBhyqmdvw/0E8Ku1NgmwxpgE0k/NXFAMOA3Eel5fPH6J0NBQrwpxuVxeb+ur1HPm7Dm1h9eWvsaXv3xJ2M1hfPr0p9xT6p5srjD76eccGLLSc0xMzBWXeXXOHVgLNDXGBBljygFFgGWec/EAzYA1wCagrjGmoDGmBBBK+sVWkRyXkpbC6ytfJ3RSKN/u+pYh9YawvtN6nwh2kazy6sjdWhttjKlHengHA92BvUCkMSYEcAFfWmtTjTETSA/6YGCQtTYhe0oXubJ9p/fRdl5bVu9bzQtVXmBUw1F65J0EFK9vhbTW9rvMcP3LrBcJRHq7H5HMcLvdTN48mX5L+hEUFMSsJ2bR5v42Tpclkuv0ISbxG0fPHaXj/I58s/MbGt/VmA9bfEj568s7XZaIIxTu4hfmuubS7ZtunE44zYSmE+hRo4emDpCApnAXn3Y8/jgvL3qZf+74J9XKVmNxm8VULVPV6bJEHKdwF5/1lesrun3TjVPnT/FGgzfoX6e/5oQR8VC4i885du4YLy96mc9//pwHb36QJW2W6Ghd5CIKd/EZyanJfLL9E/ot6cfphNOM+MsI+tXup6N1kctQuItP2HlmJ60jW/PTHz9RvVx1lrVcRpUyVZwuSyTPUrhLnpaUmsRba95i5JqRXF/weuY8N4cnKz2pO2FErkLhLnnW5kOb6Ti/I9uPbuex2x/jo+c+4qYiNzldlohPULhLnpOYksjQFUMZvWE0ZYuWZUHrBdyddreCXSQTFO6Sp2w7so02c9uw/eh2XnrgJUY3Hk2JgiUCbhpYkaxSuEueEJ8cz+srX2fMhjGUKlyKr5//mhb3tHC6LBGfpXAXx208sJG2c9uy8+ROOlbryDuN3qFU4VJOlyXi0xTu4pik1CTeWPUGEWsjuK34baxot4IGFRo4XZaIX1C4iyN2HN1Bm7lt2HpkKx2rdeS9pu9RvEBxp8sS8RsKd8lViSmJjNs4jqErh3J9weuZ33o+LU1Lp8sS8TsKd8k1rmMuWs9J/5Tpk5WeZGqLqbq9USSHKNwlx6WmpTJl8xT6LulLkZAiOloXyQUKd8lRMYdi6PpNVzYf2kzjuxozo9UMbi52s9Nlifg9hbvkiPjkeIauGMp7G9+jdJHSfPrUp7S+r7XmhBHJJQp3yXbL9iyjc3Rn9pzaQ5ewLoxqOIoSBUs4XZZIQFG4S7Y5df4UfRb34aOtH1GxZEVWtltJ/Qr1nS5LJCBlKdyNMaWBGKARkALMANzADqC7tTbNGBMOdPEsH2Gtjc5SxZLnuN1u5rjm0GNhD47HH2dAnQEMqTeEQvkLOV2aSMAK9nZDY0x+YCpw3jM0Fhhsra0LBAGtjDFlgZ5AbaAJEGGMKZC1kiUvOR5/nGe+eIZnv3iWW4rfwubOm3nrr28p2EUc5nW4A6OBKcAhz/swYJXn9SKgIVADWGetTbTWngF2AXrYpR9wu918tOUjqkyuQvRv0YxqOIrvX/qeamWrOV2aiODlaRljTHvgmLX2O2PMAM9wkLXW7Xl9FigBFAfOZNj0wvglvJ3SNSEhIeCmg3W65xMJJxi6eSgrDq2gSskqfFDrAypdX4mddmeO7dPpnp2gngNDTvXs7Tn3joDbGNMQqAbMAkpnWF4MOA3Eel5fPH6J0NBQrwpxuVxeb+urnOx5gV3AS0tfIjYxljGNx9C7Zm+Cg7LyB+C10c85MKjnzImJibniMq/C3Vpb78JrY8xKoCvwrjGmgbV2JdAMWAFsAkYaYwoCBYBQ0i+2io85EX+CV5e8yoytM6hWthornlzBvaXvdbosEbmC7LwVsg8QaYwJAVzAl9baVGPMBGAN6ef3B1lrE7Jxn5LD3G43s3+azT8W/4PTCacZWGcgwxoMIyRfiNOlicifyHK4W2sbZHh7yU3N1tpIIDKr+5Hct/vkbrp+05Wle5ZS89aafNjiQ6qUqeJ0WSJyDfQhJrlEcmoyo9eP5o3VbxCSL4RJzSfRJawL+YLzOV2aiFwjhbv8jx8P/0j7ee3ZfnQ7T4c+zYRmEyhXrJzTZYlIJincBYAjcUcYsnwIH239iDJFy2haXhEfp3APcG63m4mbJtJ/WX+SU5MJfzCckX8dSclCJZ0uTUSyQOEewOxxS/eF3Vm2dxnNKzZnXJNxVCxV0emyRCQbKNwD0In4E0SsjWDipokUzl+YSc0n0a16N821LuJHFO4BxO12M2vbLF5d8ionz5/kxSov8k6jdyhbtKzTpYlINlO4Bwh73NL1m66s/H0lj9z6CFNbTNU96yJ+TOHu5xJSEohYE8Hb696mcP7CTG0xlZcefClX5oMREeco3P3Ysj3L6PZNN3ae3MkLVV5gbOOxlClaxumyRCQXKNz90NFzR+mzuA+zf5rN3SXvZvHfF9PorkZOlyUiuUjh7kdS0lKI2hJF/6X9iUuKY0i9IQyoM0BPRRIJQAp3PxH9WzR9l/Tl1+O/Ur98fSY/NpnQmwJrXmwR+S+Fu487fPYwPRb14CvXV5hShq+e+4onKj2he9ZFApzC3Ue53W5mbJ3BPxb/g/PJ54n4awR9HulD/nz5nS5NRPIAhbsPOhB3gF6ze7FkzxLq3l6XaS2ncU+pe5wuS0TyEIW7D0lNS2XipokMWDqAfPny8UHzD+hSvYvuWReRSyjcfcQvx37hpQUvseHABuqWrcvs1rO5vcTtTpclInmUwj2PS0lL4Z117/D6qtcpFlKMj5/8mAeve1DBLiJ/SuGeh+08sZO289qy8cBGnq38LBObT6R0kdK4XC6nSxORPE7hngclpiQyev1oRqwZQaHrCvHZ05/R+r7WTpclIj7Eq3A3xuQHooAKQAFgBPALMANwAzuA7tbaNGNMONAFSAFGWGujs162/9p4YCMd53fEddzFM5WfYXzT8XqGqYhkmre3WfwdOGGtrQs0AyYCY4HBnrEgoJUxpizQE6gNNAEijDEFsl62/zl5/iQvL3yZWtNrEZcUx8IXFvLFs18o2EXEK96elvkC+DLD+xQgDFjleb8IaAykAuustYlAojFmF1AV+MHL/fqlpXuW0mlBJw7GHqRb9W5ENIygeIHiTpclIj7Mq3C31sYBGGOKkR7yg4HR1lq3Z5WzQAmgOHAmw6YXxi/h7UXChIQEn73AmJSaxHvb32PmbzO5o9gdzH50NveXup+Dew5ykINX3M6Xe/aWeg4M6jn7eH1B1RhzGzAX+MBa+6kx5p0Mi4sBp4FYz+uLxy8RGurdJFcul8vrbZ0U/Vs0vZf0Zvep3fR4qAfvNHrnmmdv9NWes0I9Bwb1nDkxMTFXXObVOXdjTBlgMfCatTbKM7zFGNPA87oZsAbYBNQ1xhQ0xpQAQkm/2BqwNh7YSNPZTXn8s8cpcF0B5reez/vN39e0vCKSrbw9ch8I3AAMMcYM8Yz1AiYYY0IAF/CltTbVGDOB9KAPBgZZaxOyWrQvyvgAjdJFSvNOw3foVbMXIflCnC5NRPyQt+fce5Ee5herf5l1I4FIb/bjD9xuN9O3TKffkn6cSz7HoLqD6F+nP0VDijpdmoj4MX2IKQf9cPAHen7bk40HNlK/fH2mtJhCpRsrOV2WiAQAhXsOiE2MZfjK4Yz/fjyli5QmqmUU7au11wM0RCTXKNyz2b9+/he9v+3NkbgjdAnrwqhGo3TPuojkOoV7Njkef5y+S/oyY+sMwm4OY17redS4pYbTZYlIgFK4Z9G5pHNMjZnKG6veIC4pjsF1BzO8wXDyBedzujQRCWAK9yxYumcpHeZ34EDsAZrc1YSxTcZS+abKTpclIqJw98a+0/sYuHwgn27/lEo3VmJlu5XUK19PF0xFJM9QuGfCuaRzTNk8hTdXv0lSahKv1X6NYfWH6dOlIpLnKNyv0df2a7pEd+Fw3GEa3tmQqS2mcucNdzpdlojIZSncr2Lf6X28tvQ1Pv/5c6qUrsK/nv0XdW6v43RZIiJ/SuF+BWcSzhCxNoJxG8cRFBTE8PrDGVB3gOaCERGfoHC/SEpaCpExkQxbOYxj8cdoU7UNIx8dyW0lbnO6NBGRa6Zw93C73SzatYhXF7+K67iL+uXrM6bxGMLKhTldmohIpincgZ/++Ik+i/uwdM9SKpasyNy/zaWVaaVbG0XEZwV0uB87d4yBywYyfct0ri94PeObjqdr9a46ry4iPi8gwz3Nnca4jeN4c/WbxCXF0btmbwbXG0zJQsD/8gMAAAU4SURBVCWdLk1EJFsEXLgfiD1Ax/kdWbJnCc3ubsa7jd7l3tL3Ol2WiEi2CphwPxJ3hOErh/PR1o8IDgpmaouphD8YrvPqIuKX/D7cU9JSmLl1Jn2X9OVc8jk6VuvIa3Veo8L1FZwuTUQkx/h1uG/Yv4G289qy6+Quat1Wi6iWUZgbjdNliYjkOL8M972n9hKxNoKoLVHcXuJ25v1tHo+bxwkOCna6NBGRXJHj4W6MCQY+AO4HEoGXrLW7cmJfu07u4q01bzFr2yzyBeeja/WujHx0JCUKlsiJ3YmI5Fm5ceT+BFDQWvuIMaYmMAZolZ072HVyF2+ufpNPfvqE/Pny06NGD/rW6sstxW/Jzt2IiPiM3Aj3OsC3ANbajcaY6tn1hVPTUnl/x/tMnzOdfEH56PVwL/rW7kvZomWzaxciIj4pN8K9OHAmw/tUY8x11tqUrH7h7Ue3M/mXyTx/3/OMbTJWoS4i4pEb4R4LFMvwPvhywe5yuTL9hQtQgNXNVnNjsRs5tf8UpziVhTJ9R0JCglffL1+mngODes4+uRHu64DHgX95zrlvv9xKoaGhXn1xl8vl9ba+Sj0HBvUcGLLSc0xMzBWX5Ua4zwUaGWPWA0FAh1zYp4hIQMvxcLfWpgFdc3o/IiLyX/pUj4iIH1K4i4j4IYW7iIgfUriLiPghhbuIiB8KcrvdTtdATEyM80WIiPigsLCwyz5xKE+Eu4iIZC+dlhER8UMKdxERP+SzT2LKzYeA5DZjTH4gCqgAFABGAL8AMwA3sAPobq1NM8aEA12AFGCEtTbaiZqzizGmNBADNCK9pxn4cc/GmAFASyCE9N/nVfhxz57f7Zmk/26nAuH48c/ZGPMwMMpa28AYczfX2KcxphAwGygNnAXaWWuPZWbfvnzk/p+HgAD9SX8IiL/4O3DCWlsXaAZMBMYCgz1jQUArY0xZoCdQG2gCRBhjCjhUc5Z5/uFPBc57hvy6Z2NMA6AW6b3UB27Dz3sGmgPXWWtrAW8AI/HTno0x/YBpQEHPUGb67AZs96w7Cxic2f37crj/z0NAgGx7CEge8AUwJMP7FCCM9KM6gEVAQ6AGsM5am2itPQPsAqrmZqHZbDQwBTjkee/vPTchfZbUucDXQDT+3/NvwHWev7yLA8n4b8+7gacyvM9Mn//JtwzrZoovh/tlHwLiVDHZyVobZ609a4wpBnxJ+v/aQdbaC7c2nQVKcOn34MK4zzHGtAeOWWu/yzDs1z0DN5J+UPIs6ZPrfUL68w78uec40k/J/ApEAhPw05+ztXYO6f95XZCZPjOOe9W7L4f7NT0ExFcZY24DVgAfW2s/BdIyLC4GnObS78GFcV/UkfSpoVcC1Uj/U7R0huX+2PMJ4DtrbZK11gIJ/O8/Yn/s+RXSe76H9OtlM0m/3nCBP/Z8QWb+DWcc96p3Xw73daSfv+PPHgLii4wxZYDFwGvW2ijP8BbPOVpIPw+/BtgE1DXGFDTGlABCSb9Q43OstfWstfWttQ2ArUBbYJE/9wysBZoaY4KMMeWAIsAyP+/5FP89Ij0J5MfPf7czyEyf/8m3DOtmii+fxvDnh4AMBG4AhhhjLpx77wVMMMaEAC7gS2ttqjFmAuk/+GBgkLU2wZGKc0YfINJfe/bcFVGP9H/gwUB3YC9+3DPwHhBljFlD+hH7QGAz/t3zBdf8+2yMmQzMNMasBZKAFzK7M31CVUTED/nyaRkREbkChbuIiB9SuIuI+CGFu4iIH1K4i4j4IYW7iIgfUriLiPghhbuIiB/6f7jz3CooD8q+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr1 = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr1,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "#implementing UCB, 1000 runs\n",
    "#CMAB for 2 arms being selected\n",
    "#factor 3/2 replaced with 1\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding ucb values. re-initialize to 0 at start of each round\n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt(math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "        \n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    N[arm1_selected] += 1\n",
    "    N[arm2_selected] += 1\n",
    "    reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df.values[t, arm1_selected+1]\n",
    "    rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD0CAYAAABgk2Y8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dZ2BU1brG8X9Cl65SDlgCqMuIUowCCigoHREbIh6QZugKwhVEuiJFECEHaVGaBa/SCVKkNwWJjcCwkGJBFDCU0BJIMvdDwrmRIslkkp2ZeX6fZtbsPft9M+HJYs/M2kFutxsREfEvwU4XICIi3qdwFxHxQwp3ERE/pHAXEfFDCncRET+kcBcR8UO5nS4AIDo6Wp/HFBHxQFhYWNCVxnNEuAOEhYV5tJ/L5SI0NNTL1eRs6jkwqOfAkJmeo6Ojr/qYTsuIiPihdM3cjTHVgdHW2jppxp4HXrLWPpB6PxzoDCQCw621UcaYAsBHQEngFNDWWnvUuy2IiMilrjlzN8b0Bd4H8qcZqwJ0BIJS75cGXgZqAg2BkcaYfEBXYIe1tjYwGxjo7QZERORy6Tktsw946uIdY8wNwCigV5ptqgGbrbUJ1tqTwF6gElALWJ66zTKgnjeKFhGRf3bN0zLW2nnGmBAAY0wu4APgFeBcms2KACfT3D8FFL1k/OLYFblcrozU/V/x8fEe7+ur1HNgUM+BIat6zuinZcKA24HJpJymucsYMx5YAxROs11h4AQQl2b84tgVefpusd5dDwzqOTCo54z5p0/LZCjcrbXbgIoAqbP5T621vVLPub9ljMkP5ANCgRhgM9AE2AY0BjZ6UL+IiGSQVz4Kaa39E4ggJbzXAAOstfGkzPArGmM2AZ2AYd44noiIr1v20zLMRMOyX5dlyfOna+Zurf0ZqPFPY9baSCDykm3OAi0yW6SIiL84e+Esr658lUnbJ1GxREUq3VApS46jLzGJiGST6EPR3Dv1XiZtn8QrNV5he6ftlC1YNkuOlWOWHxAR8VeJyYmM3jSaoeuHUqpgKVa1WcWj5R/N0mMq3EVEstD+4/tps6ANW37bQsuKLZnUdBLXF7g+y4+rcBcRyQJut5sZ38+g5/Ke5ArKxUdPfsTz9zxPUNAVF3H0OoW7iIiXHYw7yEvLXmLh7oXUCanDrCdmcUvRW7K1BoW7iIiXuN1upn83nV4renEh6QJv13ubPg/2ITgo+z+7onAXEfGCI2eOEL4knMV2MXVD6vLB4x9Qrng5x+pRuIuIZEKyO5mJ2yYyeO1g4hPjGddgHD1r9HRktp6Wwl1ExEO/nvyV9ovas+bAGhrd1ohxDcYRWiJnrI2jcBcRySC3203kt5H0Wt6LXMG5eL/Z+3So2iHbPgmTHgp3EZEMOBF/gh5f9ODjHR/ToEIDpj42lZBiIU6XdRmFu4hIOq05sIa2C9vyx6k/GFZnGANqDyBXcC6ny7oihbuIyDUkJCYwcM1Axn41FnOD4auOX3F/2fudLusfKdxFRP7B9kPbabOgDbv/2k2XsC680/AdrstzndNlXZPCXUTkCuIS4hj/9XiGbxhO6UKl+eL5L2h8e2Ony0o3hbuIyCVW7F3BCwtf4MiZIzxz1zNMfWxqtiz25U0KdxGRVGcvnKXvl31575v3uLvk3US1isrx59avRuEuIgJ898d3tJrXChtr6Vm9JyMfHUmBPAWcLstjCncRCWjJ7mTe/epd+q/uT4mCJbLlQhrZQeEuIgHr97jfabuwLasPrOaJO5/g/Wbvc8N1NzhdllekK9yNMdWB0dbaOsaYKsB/gCQgAXjBWnvYGBMOdAYSgeHW2ihjTAHgI6AkcApoa609mhWNiIik18ULafRe0ZsLyReIbBZJx6odc9TyAZl1zWXLjDF9gfeB/KlDE4CXrLV1gPlAP2NMaeBloCbQEBhpjMkHdAV2WGtrA7OBgV7vQEQkA/Ye20u9D+vRcXFHKpeuzA9dfuDFe1/0q2CHdIQ7sA94Ks3956y136fezg3EA9WAzdbaBGvtSWAvUAmoBSxP3XYZUM8rVYuIZFBiciJvb36beybfw/ZD25nSdApr267ltutvc7q0LHHN0zLW2nnGmJA09/8AMMY8CPQAHiJltn4yzW6ngKJAkTTjF8dERLLVvmP7aLuwLZt/28yTdz7JxCYTKVO4jNNlZSmP3lA1xrQEBgBNrbVHjTFxQOE0mxQGTgBpxy+OXZHL5fKkFOLj4z3e11ep58CgnjPv9IXTTIyZyMd7P6ZA7gKMrj6aZrc24+TBk5z823zUOVn1Omc43I0xrUl547SOtfZY6vA24C1jTH4gHxAKxACbgSapjzcGNl7teUNDPVvg3uVyebyvr1LPgUE9e87tdrNg9wJ6re7FwbiDdArrxMCHBnJTkZu8UKV3Zabn6Ojoqz6WoXA3xuQCIoBfgfnGGID11tohxpgIUsI7GBhgrY03xkwGZhljNgHngec96kBEJJ3iEuJ4adlLzP5hNhVLVGRLxy3UuKmG02Vlu3SFu7X2Z+DiT+eKCyxYayOByEvGzgItMlGfiEi6rd6/mg6LO3Aw7iBDHh7CoIcG5dj11rOavsQkIj7v3IVz9P2yLxO/mYi5wbClwxaq31Td6bIcpXAXEZ8WcySGVvNaEXMkhl7VezHi0RE+vSaMtyjcRcQnJbuTmbhtIv1W9aNIviIs+/cyGt3WyOmycgyFu4j4nF9P/kq7he1Y+/NamtzehOmPT6dUoVJOl5WjKNxFxGe43W5m/zCbl5e/TLI72S/XhPEWhbuI+ISjZ47SOaozC3YvoNYttZj1xCzKFy/vdFk5lsJdRHK8jb9spOXclsSei+Xtem/T+4HeAfsRx/RSuItIjpWQmMDANQN59+t3KVe8HMv+vYzKpSs7XZZPULiLSI70w58/0GZBG3Yc2UH4veGMqT+Govm19mB6KdxFJEe5kHSBMVvGMHTdUK4vcD1LWi3hsTsec7osn6NwF5EcY9fxXbT+oDXf/vEtLe5qwaSmk7jxuhudLssnKdxFxHGnz59m8NrBTNg6gRLXlWDes/N4KvSpa+8oV6VwFxFHbT+0nVbzWrH32F5aVmjJlGemUCx/MafL8nkKdxFxREJiAmO2jOGN9W9QqlAp1rVdR8lzJRXsXqJwF5Fst2r/Krot7cZPx37i2YrPMrnpZK4vcH3AXXkqKyncRSTbHD59mN4re/PJjk+47frbWNF6BQ0qNHC6LL+kcBeRLOd2u/nox4/otaIXp8+fZsjDQ3it1mvkz53f6dL8lsJdRLLUryd/pUtUF5btXcaDNz/IB49/wJ033ul0WX5P4S4iWSLZncyU7VPot6ofbrebiEYRdK/WneCgYKdLCwgKdxHxOvuXpVNUJzb8soH65eszrdk0QoqFOF1WQFG4i4jXXEi6wJsb3mTExhEUyluIDx7/gPZV2mu9dQekK9yNMdWB0dbaOsaY24CZgBuIAbpba5ONMeFAZyARGG6tjTLGFAA+AkoCp4C21tqjWdCHiDjM/mV5YeELbPt9G20rt2V0vdG6OpKDrnnyyxjTF3gfuPi29jhgoLW2NhAENDfGlAZeBmoCDYGRxph8QFdgR+q2s4GB3m9BRJyUlJzEhK8nUHVqVfYe28tnz3zGzCdmKtgdlp53NvYBaRd5CAPWp95eBtQDqgGbrbUJ1tqTwF6gElALWH7JtiLiJ3Ye2UmtGbXotaIXdcvVJaZrDC0qtnC6LCEdp2WstfOMMSFphoKste7U26eAokAR4GSaba40fnHsijz9Zlp8fHzAfatNPQeGnNzz+aTzRO6OZKprKoVyF2JU9VE0u6UZJw6e4AQnPH7enNxzVsmqnj15QzU5ze3CwAkgLvX2P41fHLui0NBQD0pJ+aPg6b6+Sj0Hhpza847DO2g9vzUxR2J47u7niGgUQYmCJbzy3Dm156yUmZ6jo6Ov+pgnHzj9zhhTJ/V2Y2AjsA2obYzJb4wpCoSS8mbrZqDJJduKiA86n3SeiK0R3B95P0fPHGXxc4uZ8/QcrwW7eJcnM/c+QKQxJi/gAuZaa5OMMRGkhHcwMMBaG2+MmQzMMsZsAs4Dz3urcBHJPl/99hUdF3fE9ZeLJrc3YUbzGZQsWNLpsuQfpCvcrbU/AzVSb+8BHr7CNpFA5CVjZwG9uyLio5LdyYzcOJJBawdxc9GbWfTcIprd0UyfW/cB+hKTiFzRbyd/I3xJOCv2reD5e55nStMpFM5X+No7So6gcBeRv3G73cz+YTYvL3+ZpOQkJjWZRJf7umi27mMU7iLyX0fOHKFzVGcW7l5IrVtqMbP5TCpcX8HpssQDCncRAWCBawGdozpzMuEkY+qP4ZUar5ArOJfTZYmHFO4iAe6vs3/x0rKX+DTmU6qWrsqaJ9dwd8m7nS5LMknhLhLAlu9dTvtF7Yk9G8vQh4fSv3Z/8ubK63RZ4gUKd5EAdOb8Gfp+2ZdJ2ydxd8m7Wf7v5VQuXdnpssSLFO4iAWbrwa20WdCGvcf20ueBPgx/ZLiuZeqHFO4iASIxOZHhG4YzfMNwyhQuw+oXVlO3XF2ny5IsonAXCQB7j+2lzYI2fH3wa9pUakNE4wiK5S/mdFmShRTuIn7M7XYzLXoavVf2Jm+uvMx5eg7P3f2c02VJNlC4i/ip/cf303VpV1buW0n98vWZ3nw6NxW5yemyJJso3EX8zIWkC4z7ahzD1g8jd3BuJjaeSNf7uxIc5MkK3+KrFO4ifmTdz+voubwnPx7+kSfvfJL/NP4PZYuUdboscYD+lIv4gcTkRPqv6k/dWXU5GX+S+c/OZ37L+Qr2AKaZu4iP23pwKx0Xd2Tn0Z2E3xvOhEYTKJCngNNlicM0cxfxUecunGPgmoHUmlGL0+dPM//Z+UxrNk3BLoBm7iI+ac2BNXRd2pU9sXtoXak1EY0iKF6guNNlSQ6imbuID0lMTmTYumHUm10Pt9vNytYr+fDJDxXschnN3EV8xM4jO2m3qB3bD22nTaU2TG46mYJ5CzpdluRQCneRHC4xOZF3trzD4HWDKZKvCJ898xnP3PWMLnsn/8ijcDfG5AFmASFAEhAOJAIzATcQA3S31iYbY8KBzqmPD7fWRmW+bJHAYP+ytF3Ylq2/b+Xp0KeZ1HQSJQuWdLos8QGennNvAuS21j4IvAG8BYwDBlprawNBQHNjTGngZaAm0BAYaYzJl/myRfzbhaQLTN89nSpTq/DTsZ+Y8/QcPm/xuYJd0s3T0zJ7gNzGmGCgCHABqAGsT318GdCAlFn9ZmttApBgjNkLVAK+yVTVIn5s55GdtF7Qmu///J5mdzRj6mNT+VfhfzldlvgYT8P9NCmnZHYDNwKPAQ9Za92pj58CipIS/CfT7Hdx/DIul8ujQuLj4z3e11epZ/+UmJzI7D2ziYiJoFCeQoy9fyyNQxpz4uAJTnDC6fKyRSC8zpfKqp49DfdXgBXW2v7GmJuBNUDaCy8WBk4Acam3Lx2/TGhoqEeFuFwuj/f1VerZ//x84mdazWvF1we/prlpztTHpnLst2N+3fOV+PvrfCWZ6Tk6Ovqqj3l6zv04/z8jPwbkAb4zxtRJHWsMbAS2AbWNMfmNMUWBUFLebBURUtZb/+DbD6g0uRK7ju7ik6c+YUHLBZQqVMrp0sTHeTpzfxeYbozZSMqM/XVgOxBpjMkLuIC51tokY0wEKUEfDAyw1sZ7oW4Rn3fo1CHCl4TzxU9fUCekDjOazyCkWIjTZYmf8CjcrbWngWev8NDDV9g2Eoj05Dgi/sjtdjMnZg49vuhBfGI8EY0i6F6tu9ZbF6/Sl5hEstGRM0fotrQb81zzqHFTDWY9MYs7brjD6bLEDyncRbKB2+3mwx8/pNfyXpy5cIbR9UbT54E+5ArO5XRp4qcU7iJZbP/x/XRb2o0V+1ZQ8+aaRDaLJLREYH0iRLKfwl0ki7jdbt7/9n16r+xNcFAw4xuOp0e1HpqtS7ZQuItkgZ9if6JTVCfW/byOR8o9wozmM7il6C1OlyUBROEu4kVJyUm8+/W7DFo7iHy58jHtsWl0vLejPgkj2U7hLuIlMUdi6LCoA98c+obHzeNMbjqZMoXLOF2WBCiFu0gmnU86z6hNoxi+YThF8xdlztNzaFmxpdZbF0cp3EUyYfuh7XRY1IEdR3bQ6u5WTGg0gRIFSzhdlojCXcQT5y6cY+i6oYz9aiylCpZi0XOLeNw87nRZIv+lcBfJoE2/bqLj4o7sid1Dx6odGdtgLMXyF3O6LJG/UbiLpNPp86fpv6o/733zHrcWu5Uv23xJvfL1nC5L5IoU7iLp8OW+L+kU1YlfTvxCj2o9GPHoCArlLeR0WSJXpXAX+Qcn4k/QZ0Ufpn8/HXODYWP7jdS8pabTZYlck8Jd5CpW7ltJx8Ud+ePUH7xW8zWG1BlC/tz5nS5LJF0U7iKXOHLmCANWD+D9794n9MZQ5necz/1l73e6LJEMUbiLpHK73Xwa8yk9lvUgLiGOVx98lTfqvqHZuvgkhbsI8OfpP+m6tCsLdy+ketnqTG8+nbtK3OV0WSIeU7hLQEs7Wz9z/gxv13ub3g/01rK84vMU7hKwLp2tz2g+QxfREL+hcJeA9PnOz+mytItm6+K3PA53Y0x/4HEgLzAJWA/MBNxADNDdWptsjAkHOgOJwHBrbVRmixbx1F9n/6Lr0q7M3TWXamWrMbP5TM3WxS95dAUBY0wd4EGgJvAwcDMwDhhora0NBAHNjTGlgZdTt2sIjDTG5PNC3SIZtnr/aipNrsRiu5gRj4xgc4fNCnbxW57O3BsCO4AFQBHgVSCclNk7wDKgAZAEbLbWJgAJxpi9QCXgm8wULZIR5y6cY/Dawbzz1TvcccMdfPHvL6hSuorTZYlkKU/D/UbgVuAxoBywGAi21rpTHz8FFCUl+E+m2e/i+GVcLpdHhcTHx3u8r69Sz+njdrtZ98c63v7+bX45/QstyregX5V+5DueD9fxnP/z0+scGLKqZ0/DPRbYba09D1hjTDwpp2YuKgycAOJSb186fpnQUM/+e+xyuTze11ep52s7ff40nZZ0Yk7MHCoUr8CqNqt4tPyjWVih9+l1DgyZ6Tk6Ovqqj3l61d5NQCNjTJAxpgxQEFidei4eoDGwEdgG1DbG5DfGFAVCSXmzVSTLrP95PWHTwvjfnf/Lm3XfZFf3XT4X7CKZ5dHM3VobZYx5iJTwDga6AweASGNMXsAFzLXWJhljIkgJ+mBggLU23juli/xdQmICb6x/g5GbRlKueDm+bPMlj5R7xOmyRBzh8UchrbV9rzD88BW2iwQiPT2OSHp8+8e3tFvYjh1HdtC+SnsiGkdovXUJaPoSk/i080nneXP9m4zcNJISBUuw+LnFNDPNnC5LxHEKd/FZaWfrbSq1YXyj8Vxf4HqnyxLJERTu4nOOnzvOsPXDmLhtIiULltRsXeQKFO7iMy6u4NhnZR+OnDlC+yrtebv+2xQvUNzp0kRyHIW7+IQTCSd46rOnWLh7IWH/CmNJqyWElQlzuiyRHEvhLjneot2L6LyyM8cSjjG2/lh61eilFRxFrkHhLjnWoVOHeGnZS8x3zeeOoncQ1TqK+8rc53RZIj5B4S45TrI7mWnR0+i3qh8JiQmMeGQETa9vSqUylZwuTcRnKNwlR3EdddEpqhObft3EI+UeYUrTKdx+w+0Bt5iUSGYp3CVHSEpOYsyWMQxZN4SCeQoyo/kM2lZuS1BQkNOlifgkhbs4btOvm3hlxStsP7SdZ+56hveavEfJgiWdLkvEp3m6KqRIpp1POk//Vf15aMZDHD1zlFlPzOKzZz5TsIt4gWbu4ojoQ9G0X9SeHUd28GLVFxnXcByF8xW+9o4iki4Kd8lW8YnxDFs3jDFbxlCqUCktHSCSRRTukm22/LaFDos6YGMtHat2ZGyDsRTLX8zpskT8ksJdstyZ82cYsGYAEVsjuLnozaxsvZL6Feo7XZaIX1O4S5Zae2AtLy55kf3H99Ptvm6MqjdK59ZFsoHCXbLE/uP7GbR2EJ/s+IQKxSuwru06Hg657EJdIpJFFO7iVcnuZCZ9M4lXv3yVIILoX6s/Ax8ayHV5rnO6NJGAonAXr9l+aDs9vujB1t+30uT2Jkx7bBpli5R1uiyRgKRwl0w7n3SeYeuGMWrzKEpcV4JZT8yiTaU2WjpAxEGZCndjTEkgGqgPJAIzATcQA3S31iYbY8KBzqmPD7fWRmWqYslRYo7E0Hp+a344/AMdqnRgXMNxFM1f1OmyRAKex8sPGGPyAFOBc6lD44CB1traQBDQ3BhTGngZqAk0BEYaY/JlrmTJCZKSkxi7ZSxh08I4dOoQC1su5IPmHyjYRXKIzMzcxwJTgP6p98OA9am3lwENgCRgs7U2AUgwxuwFKgHfZOK44rADxw/QblE7NvyygSfufIKpj03VejAiOYxH4W6MaQcctdauMMZcDPcga6079fYpoChQBDiZZteL45fxdL3u+Pj4gFvr26me3W438w/MZ+T3IwkiiBHVRtD81ubE/hpLLLFZemy9zoFBPXuPpzP3DoDbGFMPqALMBtJO3QoDJ4C41NuXjl8mNDTUo0JcLpfH+/oqJ3o+fPow4UvCWbJnCXVC6jCz+UxuLXZrth1fr3NgUM8ZEx0dfdXHPAp3a+1DF28bY9YBXYAxxpg61tp1QGNgLbANeMsYkx/IB4SS8mar+IhkdzLzds2j2xfdOJVwinENxtGzRk+Cg7RatEhO5s2PQvYBIo0xeQEXMNdam2SMiQA2kvLm7QBrbbwXjylZaOMvG+kU1Yndf+2maumqfPjkh1QsWdHpskQkHTId7tbaOmnuXvb9cmttJBCZ2eNI9km70FdIsRA+evIjnq34LHly5XG6NBFJJ32JSf5mwy8b6LCoA/uO76PH/T0YWW8khfIWcrosEckghbsAEHs2lle/fJUZ38+gfPHyrG27ljohdZwuS0Q8pHAPcG63mw9//JA+K/twIv4EfR/sy+CHB1Mwb0GnSxORTFC4B7A9sXvoEtWFtT+v5YGbHmDqY1O5p9Q9TpclIl6gcA9ACYkJjNo0ihGbRlAgdwGmNJ1CeFi4Pt4o4kcU7gFm3c/r6BLVBRtraXV3K8Y1HEfpQqWdLktEvEzhHiBiz8byP1/+DzO/n0m5YuVY/u/lNLytodNliUgWUbgHgPmu+XRd2pVj547pykgiAULh7sdijsQwdN1Q5rnmUbV0VVa2Xknl0pWdLktEsoHC3Q9dSLrA2C1jGbp+KLmDczO87nD61uyrb5iKBBCFu59Z//N6un/RnZ1Hd/LMXc8wuelkbrzuRqfLEpFspnD3E8fOHaP3it7M+mEWtxa9lUXPLeJx87jTZYmIQxTufmDR7kV0WdqFo2eO6g1TEQEU7j7t97jf6bK0C1F7oqhcqjJfPP8FVf9V1emyRCQHULj7qKg9UbRb2I5ziecYXW80vWr0Im+uvE6XJSI5hMLdxxw+fZiB3wxk/oH5VCldhU+f/hRzo3G6LBHJYRTuPiIxOZGIrREMWTeE+Avx9KvZj2F1hpEvdz6nSxORHEjh7gN+PPwj7Re159s/vqXp7U3pflt3Gldr7HRZIpKDaRnAHCwxOZFRm0Zx37T7+D3udz5v8TlLWi0hpHCI06WJSA6nmXsO9d0f39FlaRe2/b6NFne1YFLTSfoykoikm8I9hzkRf4I317/J+K3jufG6G5nz9Byeu/s5p8sSER/jUbgbY/IA04EQIB8wHNgFzATcQAzQ3VqbbIwJBzoDicBwa21U5sv2T5/t/IxuS7sRey6WzmGdGfnoSIoXKO50WSLigzw9594aiLXW1gYaAxOBccDA1LEgoLkxpjTwMlATaAiMNMbo4x2X+PP0n7T4vAUt57akfPHyRHeKZspjUxTsIuIxT0/LfA7MTXM/EQgD1qfeXwY0AJKAzdbaBCDBGLMXqAR84+Fx/UqyO5nxX49nwJoBuN1uRjwygldrvkruYJ0tE5HM8ShFrLWnAYwxhUkJ+YHAWGutO3WTU0BRoAhwMs2uF8cv43K5PCmF+Ph4j/d10l/xf/H6ttfZ9Ocm6papy6uVXyWkcAg/2Z+uua+v9pwZ6jkwqGfv8XiKaIy5GVgATLLWfmKMeTvNw4WBE0Bc6u1Lxy8TGhrqUR0ul8vjfZ3gdruZ/cNs+q7uS1xCHFOaTqFTWCeCgoLS/Ry+1rM3qOfAoJ4zJjo6+qqPeXTO3RhTClgJ9LPWTk8d/s4YUyf1dmNgI7ANqG2MyW+MKQqEkvJma0DaeWQnjT9uTLtF7QgpFsL28O10vq9zhoJdRCQ9PJ25vw4UBwYZYwaljvUEIowxeQEXMNdam2SMiSAl6IOBAdba+MwW7WtijsQwcM1AFtvFFMpbiImNJ9L1/q4EB+k7ZCKSNTw9596TlDC/1MNX2DYSiPTkOL4uLiGOYeuGMWHrBIrkK8KA2gPoVaMXN1x3g9OliYif08cysoDb7Wburrn0WtGLP079Qfi94Yx4dIRCXUSyjcLdy/Yd20ePZT1Yvnc5VUtXZUHLBVQrW83pskQkwCjcvSQhMYExW8bw1sa3yBOchwmNJtDt/m76zLqIOELJ4wVrDqyh29Ju2FjLsxWf5d2G71KmcBmnyxKRAKZwz4TDpw/TZ2UfPt7xMRWKV2D5v5fT8LaGTpclIqJw90RiciLvbXuPIeuGcC7xHIMfGsxrtV6jQJ4CTpcmIgIo3DMkMTmRyOhIxm8dz57YPTSs0JAJjSboGqYikuMo3NPB7XYTtSeKQWsH8cPhH6hetjrzn53PE3c+oW+XikiOpHC/hl1Hd9FzeU9W7V9F+eLlmdtiLk+FPqVQF5EcTeF+FacSTvHmhjd59+t3KZy3MBGNIuhyXxfy5MrjdGkiItekcL+CDb9s4IUFL/DLyV/oWLUjo+qN0vVLRcSnKNzT+PP0n4zaNIqIrRFUuL4Cmzts5sGbH3S6LBGRDFO4k/Lt0vFfj2f4xuGcvXCW8HvDeafhOxTKW8jp0kREPBLQ4e52u1lsF9NnZR/2Hd9Hc9OcMfXHcPsNtztdmohIpgRsuMcciaHX8l6sPrCau0rcxcrWK6lfob7TZYmIeEXAhXvs2ViGrBvC5C/x/8QAAAUgSURBVO2TKZqvKP9p/B+63NdFC3yJiF8JmEQ7GX+SKdunMHrzaOIS4uh2XzeG1hmqNdZFxC8FRLgvtovpurQrh04dom5IXSIaR3B3ybudLktEJMv4dbi7jrrou6ovUXuiuKfkPbpwhogEDL8M922/b2PUplEs2bOEgnkKMurRUbzywCvkzZXX6dJERLJFloe7MSYYmARUBhKAF621e7PiWAfjDvL66tf58McPKZ6/OF3CujD44cGUKFgiKw4nIpJjZcfM/Qkgv7X2AWNMDeAdoLk3D3Dg+AFGbhrJzO9n4sbN67Vep1+tfhTJV8SbhxER8RnZEe61gOUA1tqvjTH3eeuJk93JTIyZyPT503G73bx474v0rdmXkGIh3jqEiIhPyo5wLwKcTHM/yRiT21qbmNkn/vHwj0zaNYmWFVvyToN3KFukbGafUkTEL2RHuMcBhdPcD75SsLtcrgw/cT7ysb7xekoULkHc73HE/R6XiTJ9R3x8vEc/L1+mngODevae7Aj3zUAz4LPUc+47rrRRaGioZ8/uysS+PsrlcqnnAKCeA0Nmeo6Ojr7qY9kR7guA+saYLUAQ0D4bjikiEtCyPNyttclAl6w+joiI/L9gpwsQERHvU7iLiPghhbuIiB9SuIuI+CGFu4iIHwpyu91O10B0dLTzRYiI+KCwsLCgK43niHAXERHv0mkZERE/pHAXEfFDPnslpuy8CEh2M8bkAaYDIUA+YDiwC5gJuIEYoLu1NtkYEw50BhKB4dbaKCdq9hZjTEkgGqhPSk8z8eOejTH9gceBvKT8Pq/Hj3tO/d2eRcrvdhIQjh+/zsaY6sBoa20dY8xtpLNPY0wB4COgJHAKaGutPZqRY/vyzP2/FwEBXiPlIiD+ojUQa62tDTQGJgLjgIGpY0FAc2NMaeBloCbQEBhpjMnnUM2ZlvoPfypwLnXIr3s2xtQBHiSll4eBm/HznoEmQG5r7YPAG8Bb+GnPxpi+wPtA/tShjPTZFdiRuu1sYGBGj+/L4f63i4AAXrsISA7wOTAozf1EIIyUWR3AMqAeUA3YbK1NsNaeBPYClbKzUC8bC0wBDqXe9/eeG5KySuoCYAkQhf/3vAfInfo/7yLABfy3533AU2nuZ6TP/+Zbmm0zxJfD/YoXAXGqGG+y1p621p4yxhQG5pLyVzvIWnvxo02ngKJc/jO4OO5zjDHtgKPW2hVphv26Z+BGUiYlLUhZXO9jUq534M89nybllMxuIBKIwE9fZ2vtPFL+eF2UkT7TjnvUuy+He7ouAuKrjDE3A2uBD621nwDJaR4uDJzg8p/BxXFf1IGUpaHXAVVI+a9oyTSP+2PPscAKa+15a60F4vn7P2J/7PkVUnq+g5T3y2aR8n7DRf7Y80UZ+Tecdtyj3n053DeTcv6Of7oIiC8yxpQCVgL9rLXTU4e/Sz1HCynn4TcC24Daxpj8xpiiQCgpb9T4HGvtQ9bah621dYDvgReAZf7cM7AJaGSMCTLGlAEKAqv9vOfj/P+M9BiQBz//3U4jI33+N9/SbJshvnwaw58vAvI6UBwYZIy5eO69JxBhjMkLuIC51tokY0wEKS98MDDAWhvvSMVZow8Q6a89p34q4iFS/oEHA92BA/hxz8C7wHRjzEZSZuyvA9vx754vSvfvszFmMjDLGLMJOA88n9GD6RuqIiJ+yJdPy4iIyFUo3EVE/JDCXUTEDyncRUT8kMJdRMQPKdxFRPyQwl1ExA8p3EVE/ND/AVqeGnhh//cBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr2 = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr2,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "#implementing UCB, 1000 runs\n",
    "#CMAB for 2 arms being selected\n",
    "#factor 3/2 replaced by 3\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding the ucb values. we pick the max \n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt(3*math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "        \n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    N[arm1_selected] += 1\n",
    "    N[arm2_selected] += 1\n",
    "    reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df.values[t, arm1_selected+1]\n",
    "    rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD0CAYAAABgk2Y8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deYDN9f7H8edgjKVBJWsLFZ87JUsqW5Yru6JUirINQiRyKSKXkCkRWWJSksp17WFQWWeyTv0KnT7oVtI6yFhnzMw5vz9m3DsVZc6cme9ZXo+/zvl8z3e+7/ecmdd85/P9nu83zOPxICIiwaWA0wWIiIjvKdxFRIKQwl1EJAgp3EVEgpDCXUQkCCncRUSCUCGnCwBITEzU+ZgiIl6oXbt22IXG/SLcAWrXru3Vei6Xi6ioKB9X49/Uc2hQz6EhNz0nJiZedJmmZUREgpDCXUQkCCncRUSCkMJdRCQIKdxFRIKQwl1EJAgp3EVEgpDCXUTEAXEH4jDTDXGH4vLk6yvcRUTy0alzp+i7qi9t3m1DRMEIql9ZPU+2o3AXEcknO7/fSa3ZtZiTOIeh9Yeyq/cuKhavmCfb8pvLD4iIBKsMdwYvffwSozaOokJkBTZ130Sj6xrl6TYV7iIieeiTHz+hx4oefP7z53S8uSOz755NqSKl8ny7mpYREckDaRlpjNk0hjqv1+HImSP864F/sfD+hfkS7KA9dxERn9t/dD9dlnVh5/c7ebT6o0xtNZUril6RrzUo3EVEfMTj8TA7cTZD1g8homAEix5YxIM3P+hILQp3EREfSDqdRM+VPXl///s0v745b7Z/k4ol8uZMmEuhcBcRyQWPx8O7e95lyPoh/JryK1NaTmFgnYEUCHP2kKbCXUTESydST9BjRQ+WupZya/lbWd9lPdXL5s2HknJK4S4i4oX4Q/H0WtmLg8cOEtMshqfqPUWhAv4TqZdUiTGmDhBjrW1ijKkJvApkAKlAV2vtz8aY3kAfIB0YZ61dZYwpCiwAygAngW7W2qS8aEREJD9kuDOYGD+R5zY9xzUlrmF9l/U0rdzU6bL+4C8nhYwxw4DXgSJZQ1OBJ6y1TYClwNPGmHLAQKAB0BJ4wRgTAfQD9lhrGwLzgZE+70BEJJ98l/wdd82/i5EbR/LQzQ+xp98evwx2uLQPMX0FdMj2/GFr7f9lPS4EpAB3AAnW2lRrbTJwEKgO3AmszXptHNDMJ1WLiOSzJV8socZrNdj9w27mtZ/HOx3eITIi0umyLuovp2WstUuMMZWyPf8RwBhTHxgANCJzbz0522ongZJAiWzj58cuyOVy5bD0TCkpKV6vG6jUc2hQz/7h19Rfifm/GFZ+u5Jql1fjpbovcV3EdXz55Zc++fp51bNXs//GmIeAZ4G21tokY8wJIPufsEjgOJB9/PzYBUVFRXlTCi6Xy+t1A5V6Dg3q2VnnT3Ec9MEgklOSGdVoFCMbjaRwwcI+3U5uek5MTLzoshyHuzHmUTIPnDax1h7LGt4JjDfGFAEigChgL5AAtMla3hrYmtPtiYjktyNnjtBteTfWHFhDnYp1eL3d61QrU83psnIkR+FujCkITAMOAUuNMQCbrbWjjTHTyAzvAsCz1toUY8ws4C1jTDxwDujs0+pFRHxs8zebeWTpIxw5c4SprabS//b+FCxQ0OmycuySwt1a+w1QN+vpBa9+Y62NBWJ/N3YGcObCCiIiOZDuTmfUhlFMTJhIlSuqsK3nNmqVr+V0WV7znzPuRUQc8v2J7+m0pBNbD22lV61evNLqFYoXLu50WbmicBeRkOXxeFhhV/DY+49xOu00b9/3No9Wf9TpsnxC4S4iIckesfRZ1YfN326mWplqLHpgEVFX+ceZOr6gcBeRkHIm7Qzjtoxj0seTKF64OLPazqJnrZ6EFwx3ujSfUriLSMiIOxBHv9X9+Db5W7rV6MaLzV+kTPEyTpeVJxTuIhL0Tp07xdD1Q3kt8TVuuuomNnffTKPrGjldVp5SuItI0PJ4PCzat4gh64fww8kfGFJvCOOajqNIoSJ/vXKAU7iLSFBKOp1Er/d7sdKupFa5Wix6cBH1r6nvdFn5RuEuIkFn++HtPPjvB0k6ncSk5pMYVHdQQH7KNDcU7iISNFLTU5kYP5HxW8dzdYmr+bjnx9xa/lany3KEwl1EgsKmbzbRd1Vf7FHLw9UeZmabmVxe9HKny3KMs7fnFhHJpXR3Ov9Y/w/+/tbfOZdxjrWPrOW9+98L6WAH7bmLSAA7cuYIDy1+iA1fb6Dfbf2Y1GISxcKLOV2WX1C4i0hAWmlX0ndVX46dPcab7d+ke83uTpfkVzQtIyIBJd2dzjMfPkP7he0pU7wM23puU7BfgPbcRSRguJJcPLbqMeIPxdOndh+mtppKRKEIp8vySwp3EfF76e50xm8ZT0xCDMXCi/HWvW/RtUZXp8vyawp3EfFrh5IP0XlJZxK+S+CBmx5gWqtplI8s73RZfk/hLiJ+a/mXy4leEU2aO413OrxD51t0G+ZLpQOqIuJ3jqccp//q/tz3r/uofHllPu3zqYI9h7TnLiJ+w+1xM3X7VMZsHsPxlOMMqjOIic0m6qCpFxTuIuIXvj3+LdGbotmZtJMWN7QgplkMNcvVdLqsgHVJ4W6MqQPEWGubGGNuBOYBHmAv0N9a6zbG9Ab6AOnAOGvtKmNMUWABUAY4CXSz1iblQR8iEqA8Hg9zEufw1PqnwANz282lR80ehIWFOV1aQPvLOXdjzDDgdeD81e0nAyOttQ2BMKC9MaYcMBBoALQEXjDGRAD9gD1Zr50PjPR9CyISqL45/g0tFrSg7+q+1L+mPitariC6VrSC3Qcu5YDqV0CHbM9rA5uzHscBzYA7gARrbaq1Nhk4CFQH7gTW/u61IhLi3B43M3fNpNrMamw/vJ1ZbWex7tF1VCxe0enSgsZfTstYa5cYYyplGwqz1nqyHp8ESgIlgORsr7nQ+PmxC3K5XJdedTYpKSlerxuo1HNoCNaevz/9PSN3jWTHLzuoX7Y+Y24bQ8XiFbFf2qDt+c/kVc/eHFB1Z3scCRwHTmQ9/rPx82MXFBUV5UUpmX8UvF03UKnn0BBsPbs9bsZsGkNMQgyFCxYm9p5Yetbq+ZspmGDr+VLkpufExMSLLvPmPPdPjTFNsh63BrYCO4GGxpgixpiSQBSZB1sTgDa/e62IhJgfT/5Im3faMHbLWO6Luo89/fbQ69ZemlvPQ97suQ8BYo0xhQEXsNham2GMmUZmeBcAnrXWphhjZgFvGWPigXOAPoUgEkLOnwkzZP0Q3B43s9rOok/tPgr1fHBJ4W6t/Qaom/V4P9D4Aq+JBWJ/N3YGeDDXVYpIwPnh5A/0XNmTtQfX0rRyU15r+xpVrqzidFkhQx9iEhGfcnvcTNsxjeEfDQdgeuvpPH7749pbz2cKdxHxmUPJh+i+vDsbv9lI2yptmdxyMlWvrOp0WSFJ4S4iuebxeHj787d5Iu4J3B73Bc+EkfylcBeRXEk6nUTf1X1Z6lrKndfeyVv3vsX1l1/vdFkhT+EuIl7xeDws2reIAXEDOJF6gphmMQypN4SCBQo6XZqgcBcRL3yX/B0D1w5k+ZfLub3C7cxtN5dbyt7idFmSjcJdRC5ZWkYacxLn8PSHT5PhyeCl5i8xqO4gChVQlPgbvSMickkOnzjMI0sfYcu3W2h+fXNi74nlulLXOV2WXITCXUT+lMfj4b2979F/TX/OZZzjzfZv0q1GN50J4+cU7iJyUcfOHqPf6n4s2reIelfXY/5987nxihudLksugcJdRC5o7cG1RK+IJulMEuObjmdYg2GaWw8geqdE5DdOnzvN0A+GMmv3LG666iZWdV7FreVvdbosySGFu4j8147DO+iyrAsHjh1gcN3BTLhrAkUKFfnrFcXvKNxFhLSMNJ7f8jwTtk6gQmQFNnTdwN8r/93psiQXFO4iIc6V5KLLsi4k/phI1xpdmdZqGiWLXPSOmBIgFO4iIcrtcTN953Se/vBpiocXZ/GDi7n/pvudLkt8ROEuEoLsEUu/1f3Y+M1G2lRpw9x2cyl3WTmnyxIfUriLhJB0dzrjt4xnQvwEioUXY87dc3Qv0yClcBcJEYeSD9F5SWcSvkugU7VOTGk5hbKXlXW6LMkjCneRIHcm7QyTt01m/NbxFCpQiHc7vEunWzo5XZbkMYW7SBD77KfP6LSkE64jLu79271MaTmFSqUqOV2W5AOFu0gQSstIY/K2yYzeNJrLi17O+kfX0/yG5k6XJflI4S4SZD798VP6r+nPtsPbuPdv9zLn7jlcVfwqp8uSfOZVuBtjwoG3gEpABtAbSAfmAR5gL9DfWus2xvQG+mQtH2etXZX7skXk95JTknl2w7PM3DWTyIhIFt6/kIeqPeR0WeKQAl6u1wYoZK2tD4wFxgOTgZHW2oZAGNDeGFMOGAg0AFoCLxhjInJftohkt/P7ndScXZOZu2Yy4I4BHHzioII9xHk7LbMfKGSMKQCUANKAusDmrOVxQAsy9+oTrLWpQKox5iBQHdiVq6pFBMg8b/25jc8RkxBDxciKJEQnUO+aek6XJX7A23A/ReaUzJdAaeBuoJG11pO1/CRQkszgT8623vnxP3C5XF4VkpKS4vW6gUo9h4a/6vnwqcM8s/MZPjnyCfdXvp9hNYYReSoyoL9Pep99x9twHwyss9YON8ZcA2wACmdbHgkcB05kPf79+B9ERUV5VYjL5fJ63UClnkPDxXr2eDzM/2w+T3z4BGFhYSy4bwGPVH/EgQp9T+9zziQmJl50mbdz7r/yvz3yY0A48KkxpknWWGtgK7ATaGiMKWKMKQlEkXmwVUS8cPTMUTou7kj3Fd2pVb4Wn/f9PGiCXXzL2z33KcAbxpitZO6xjwB2A7HGmMKAC1hsrc0wxkwjM+gLAM9aa1N8ULdIyPngqw/ovqI7SaeTiGkWw5B6QyhYoKDTZYmf8ircrbWngI4XWNT4Aq+NBWK92Y6IwNm0swz/aDhTd0wlqnQUqzqtolb5Wk6XJX5OH2IS8WOf/fQZjyx9hH1J+3jijieIaRZD0fCiTpclAUDhLuKH3B43b3z5BtP2TeOKolcQ90gcrW5s5XRZEkAU7iJ+5rvk7+i6vCubvtnEfX+7jzn3zKF0sdJOlyUBRuEu4kcW7l1I31V9yfBkMO72cYxoPUI30hCvKNxF/MDxlOMMWDOAd/a8Q72r6/H2fW9z7udzCnbxmrfnuYuIj2z4egO3zLqFhXsXMrbJWLb02MINV9zgdFkS4LTnLuKQE6knGLBmAG9//jZVr6zKxz0/5o6KdzhdlgQJhbuIA5a6ljJgzQB+Of0Lw+8czshGIykWXszpsiSIKNxF8tFPp35iwJoBLHEtoWa5mix7aBl1rq7jdFkShBTuIvnA4/Hw1mdvMXjdYM6mneWFu15gSL0hhBcMd7o0CVIKd5E89tWxr+i3uh8f/OcDGl7bkNh7YjGljdNlSZBTuIvkkbNpZ3kh/gVeTHiR8ILhzGgzg7639aVAmE5Sk7yncBfJAwmHEuizqg/7kvbRqVonJrWYRIXICk6XJSFE4S7iQ7+e/ZWBawey4PMFVIiswNpH1tLyxpZOlyUhSOEu4iMbvt5A9+Xd+fHUj4xqNIqnGzxN8cLFnS5LQpTCXSSXzqSd4ZkPn+HVna9mfhgp+mNur3i702VJiFO4i+TC9sPb6ba8G/uP7mfgHQN5odkL+jCS+AWFu4gXjp09Rkx8DJO2TeLqElfzUdePaFq5qdNlifyXwl0kh+IOxBG9MpqfTv1Ej5o9mNJyCiWLlHS6LJHfULiLXKJjZ48x/MPhzPlkDtXKVGN159XcWv5Wp8sSuSCFu8hfcHvczNg5g+c2PceJ1BMMqTeEcU3HUaRQEadLE7kohbvInzh29hhdl3Vl9YHVNLquEa+0fIVa5Ws5XZbIX1K4i1zE6v2riV4ZzbGzx5jRZgb9buunOyNJwPA63I0xw4F2QGFgJrAZmAd4gL1Af2ut2xjTG+gDpAPjrLWrclu0SF46kXqC5zc/z6Rtk6hRtgbrH11PjXI1nC5LJEe8uoKRMaYJUB9oADQGrgEmAyOttQ2BMKC9MaYcMDDrdS2BF4wxET6oW8TnPB4PcxLncN0r1zFp2yT61u7L9l7bFewSkLzdc28J7AGWASWAoUBvMvfeAeKAFkAGkGCtTQVSjTEHgerArtwULeJrP578kZ4rexJ3MI6mlZsS0yyG2yrc5nRZIl7zNtxLA9cBdwOVgZVAAWutJ2v5SaAkmcGfnG298+N/4HK5vCokJSXF63UDlXr2rbjv4hibOJbUjFSerfUsnW7sRIHkAriSnf0e630ODXnVs7fhfhT40lp7DrDGmBQyp2bOiwSOAyeyHv9+/A+ioqK8KsTlcnm9bqBSz76RnJLMgLgBLPh8AXdUvIP59873q5to6H0ODbnpOTEx8aLLvL1rQDzQyhgTZoypABQHPsqaiwdoDWwFdgINjTFFjDElgSgyD7aKOCr+UDw1XqvBe3ve45+N/0lCdIJfBbtIbnm1526tXWWMaURmeBcA+gNfA7HGmMKAC1hsrc0wxkwjM+gLAM9aa1N8U7pIzqVlpDF281gmxE+gUqlKxEfHU/fquk6XJeJzXp8Kaa0ddoHhxhd4XSwQ6+12RHzFHrF0Xd6Vnd/vpEfNHkxtNZXIiMi/XlEkAOlDTBL00t3pvPzxy4zeNJpi4cX494P/5oGbHnC6LJE8pXCXoPZF0hf0WNGDnd/vpENUB2a0mUG5y8o5XZZInlO4S1DyeDzM2j2LIeuHUDy8OAvvX0jHmzvq8gESMhTuEnSOnDlCz5U9WWlX0urGVsxrP4+yl5V1uiyRfKVwl6CR4c5g1u5ZjN86nmNnjzGl5RQG1hlIgTBvz/gVCVwKdwkKu3/YTc+VPfn858+pVa4Wazqv0aV5JaQp3CWguT1uXkp4iZEbR1K2eFkWP7iYDlEdNLcuIU/hLgHr8InDdF3WlY3fbOSBmx5g9t2zuaLoFU6XJeIXFO4ScNweN298+gZDPxhKWkYac9vNpUfNHtpbF8lG4S4BJfGHRAatG0T8oXgaXdeI2HtiqXplVafLEvE7CncJCKkZqQxdP5TJ2ydTulhpYu+JJbpWtM6EEbkIhbv4vS+SvuDRDY+y79d99L61Ny82f5FSRUo5XZaIX1O4i986fe40Iz4awazdsyhSsAgrHl5BO9PO6bJEAoLCXfzS9sPb6bKsC18d+4quNbrS+7reNDANnC5LJGBowlL8yrmMc4zcMJIGbzQgLSONDd02MO/eeVxRRKc4iuSE9tzFb3yR9AVdlnXhkx8/oXvN7kxtNZUSESWcLkskICncxXFuj5up26cy/KPhREZEsrTjUu6Lus/pskQCmsJdHLXmwBrGbh7Lju93cE/Ve4i9J1ZXcBTxAYW7OOL0udMMXjeY2E9iqVSqEq/f8zrRtaL1KVMRH1G4S77b8PUG+q3ux4GjBxhWfxhj/z6WiEIRTpclElQU7pJvfj71M//44B8s+HwB119+PR90+YC7rr/L6bJEgpLCXfLFu3veZcCaAZw6d4pRjUYx/M7hFA0v6nRZIkErV+FujCkDJALNgXRgHuAB9gL9rbVuY0xvoE/W8nHW2lW5qlgCyvGU4zy++nHe2/se9a6ux9x2c4m6KsrpskSCntcfYjLGhAOzgbNZQ5OBkdbahkAY0N4YUw4YCDQAWgIvGGM0uRoCTp07xcT4iVSbWY1F+xYxtslYtvTYomAXySe52XOfBLwGDM96XhvYnPU4DmgBZAAJ1tpUINUYcxCoDuzKxXbFzyUcSiB6ZTT7j+6n8XWNWdJxCXWuruN0WSIhxatwN8Z0B5KsteuMMefDPcxa68l6fBIoCZQAkrOten78D1wulzelkJKS4vW6gcpfez6XcY45rjnMcc2hXLFyvNH4DeqWrQsnvX9/z/PXnvOSeg4NedWzt3vu0YDHGNMMqAnMB8pkWx4JHAdOZD3+/fgfREV59++6y+Xyet1A5Y897zi8gwFxA9j9w24euvkhXrv7NZ9eltcfe85r6jk05KbnxMTEiy7zKtyttY3OPzbGbAL6Ai8ZY5pYazcBrYGNwE5gvDGmCBABRJF5sFWCRHJKMr3e78XiLxZTtnhZlnRcQoeoDk6XJRLyfHkq5BAg1hhTGHABi621GcaYacBWMg/ePmutTfHhNsVBy1zLGLxuMIdPHOafjf/JU/WeIjIi8q9XFJE8l+twt9Y2yfa08QWWxwKxud2O+I/slw646aqbSIhO0AFTET+jDzFJjuz5eQ8dF3fEHrE83eBpxjUdR6EC+jES8Tf6rZRL4va4iU2MZfC6wZSIKKFLB4j4OYW7/Kl0dzobv97I6E2j2XZ4G3dVvosFHRZQ7rJyTpcmIn9C4S4XtfP7nXRf3h3XERflLyvP3HZz6VGzhy7LKxIAFO7yB6npqfxz0z958eMXqRBZgQX3LaBDVAdd6EskgCjc5TcSf0ik2/Ju7EvaR3TNaCa3nEzJIhf8ULGI+DGFuwCw75d9jN86noV7F1KmeBlWd15NmyptnC5LRLykcA9xaRlpDP9oOC9ve5nIwpE8Ve8pRjYa6dNLB4hI/lO4h6h0dzqL9i3i+S3P8+WRL+lbuy/jmo7jymJXOl2aiPiAwj3EuD1uXtv9GjEJMRxKPsTNV93MiodX0M60c7o0EfEhhXsI+c+v/2Fg3EBWH1jNndfeyfTW02lbtS0Fwry+Z4uI+CmFewhITU9lYvxEJsRPwOPx8GrrV+l/e3+dry4SxBTuQczj8fDhfz5kQNwA9h/dT6dqnZhw1wQqlarkdGkikscU7kFq23fbGPrBUBK+S6ByqcqsfWQtLW9s6XRZIpJPFO5B5mzaWUZvGs3L216m/GXlmdFmBtG1oilSqIjTpYlIPlK4B5FN32zisfcf48CxA/S+tTcvt3hZN88QCVEK9yBwIvUEQ9YN4fVPX6dyqcp82OVDXY5XJMQp3ANYanoqM3bN4MWEF0k6k8Sw+sMY3WQ0xcKLOV2aiDhM4R6g4g/F89j7j+E64uKuyncx4a4J3FHxDqfLEhE/oXAPMD+d+olndjzDym9Xcm3Ja1nTeQ2tq7R2uiwR8TMK9wCRlpHG9J3TGb1pNCnpKYy4cwQjGo6geOHiTpcmIn5I4R4A4g/F03dVX/Yl7aP1ja15osoTtL5De+sicnG6qIgfO5t2ln+s/weN3mzE6bTTrHh4Bas7r6ZSZCWnSxMRP+fVnrsxJhx4A6gERADjgC+AeYAH2Av0t9a6jTG9gT5AOjDOWrsq92UHv0PJh+jwrw4k/pjIY7c+xsstX+aywpc5XZaIBAhv99wfBY5aaxsCrYHpwGRgZNZYGNDeGFMOGAg0AFoCLxhjInJfdvA6mXqSKdumEDUjiv1H97Py4ZXMvme2gl1EcsTbOfd/A4uzPU8HagObs57HAS2ADCDBWpsKpBpjDgLVgV1ebjeoxR2Io+vyrhw5c4RWN7Zieuvp3HDFDU6XJSIByKtwt9aeAjDGRJIZ8iOBSdZaT9ZLTgIlgRJAcrZVz4//gcvl8qYUUlJSvF7XXxxPPc7ML2ay4MACqpasyitNX6HWlbU49/M5XD//sbdg6Dmn1HNoUM++4/XZMsaYa4BlwExr7bvGmBezLY4EjgMnsh7/fvwPoqKivKrD5XJ5va7TMtwZzNg1g9GbRnMi9QSP3/Y4k1pMomh40T9dL5B79pZ6Dg3qOWcSExMvuszbA6plgfXAAGvtR1nDnxpjmlhrN5E5D78R2AmMN8YUIfPAaxSZB1tD3u4fdjNgzQB2fL+DFje0YFLzSdxS9hanyxKRIOHtnvsI4HJglDFmVNbYk8A0Y0xhwAUsttZmGGOmAVvJPHj7rLU2JbdFB7KfTv3E0x8+zfzP5lO6WGne6fAOnap10l2RRMSnvJ1zf5LMMP+9xhd4bSwQ6812gsm5jHO8uuNVxmweQ0p6CsPvHM6wBsMoVaSU06WJSBDSJ1TzwdqDaxm0dhD2qKVtlbZMbjmZqldWdbosEQliCvc8ZI9Yhn4wlPf3v0+VK6qwqtMq2lZt63RZIhICFO55IC0jjUkfT2LM5jGEFwwnplkMT9Z5kohC+vyWiOQPhbuP7Ti8g97v92bPL3u492/3MqvtLMpdVs7pskQkxOjCYT6S4c5gzKYx1Jtbj2Nnj7H8oeUse2iZgl1EHKE9dx+IPxTPoLWDSPwxkS7VuzCjzQzdmFpEHKVwz4UDRw8weN1gVh9YTYXICiy8fyEdb+6oc9ZFxHEKdy98e/xbxmwew9ufv00YYcQ0i6H/7f11VyQR8RsK9xxIy0hjyvYpjNk8BrfHzeO3Pc7QBkO5usTVTpcmIvIbCvdLtPXbrfRb3Y99Sftob9ozrfU0ri15rdNliYhckML9Lxw7e4zB6wYz/7P5XFvyWlY8vIJ2pp3TZYmI/CmF+0Wku9N5/ZPXeW7jcxw7e4wRd45gRMMRmlcXkYCgcL+ATd9s4vHVj+M64qLhtQ2Z1noaNcvVdLosEZFLpnDP5vCJwzy59kmWupZSuVRllj20jPamvU5tFJGAo3AHPB4Ps3bPYtgHwwgLC2N049EMazCMYuHFnC5NRMQrIR/uu3/YzdMfPs2GrzfQ4oYWTG89nSpXVnG6LBGRXAnZcD9w9ADPfPQMy1zLuLzo5cxsM5O+t/XVFIyIBIWQC/dDyYd4fvPzzPtsHuEFwnnmzmd0RyQRCTohE+6Hkg8xe/dsJm+fDEDvW3vzXOPndNVGEQlKQR/uP5/6mbGbxxL7SSxp7jQervYwMc1i9OlSEQlqQRvuZ9POMjF+Ii9ve5nUjFR61erFoLqDMKWN06WJiOS5oAz3hEMJ9F/Tn89+/owHb3qQ8U3H6wwYEQkpeR7uxpgCwEygBpAK9LLWHsyLbe3+YTejNo5i7cG1lC1eltWdV9OmSpu82JSIiF/Lj9vs3QsUsdbWA54BXvb1Bvb+spd7F97L7bG3s+v7XcQ0i+GrgV8p2EUkZOXHtMydwFoAa+12Y8xtvvrCGe4Mpu6Zylw7l+LhxRnbZCxP1n2SEhElfLUJEZGAlB/hXgJIzspFzMsAAAP3SURBVPY8wxhTyFqbntsvvPeXvcx2zaZL9S5MaTmFK4tdmdsvKSISFPIj3E8A2e8WXeBCwe5yuXL8hQtTmC2tt1A6sjS/fPsLv/BLLsoMHCkpKV59vwKZeg4N6tl38iPcE4B7gEXGmLrAngu9KCoqyqsv7nK5vF43UKnn0KCeQ0Nuek5MTLzosvwI92VAc2PMx0AY0CMftikiEtLyPNyttW6gb15vR0RE/ic/ToUUEZF8pnAXEQlCCncRkSCkcBcRCUIKdxGRIBTm8XicroHExETnixARCUC1a9e+4L1B/SLcRUTEtzQtIyIShBTuIiJBKGDvxJSfNwHJb8aYcOANoBIQAYwDvgDmAR5gL9DfWus2xvQG+gDpwDhr7SonavYVY0wZIBFoTmZP8wjino0xw4F2QGEyf543E8Q9Z/1sv0Xmz3YG0Jsgfp+NMXWAGGttE2PMjVxin8aYosACoAxwEuhmrU3KybYDec89z28C4qBHgaPW2oZAa2A6MBkYmTUWBrQ3xpQDBgINgJbAC8aYCIdqzrWsX/zZwNmsoaDu2RjTBKhPZi+NgWsI8p6BNkAha219YCwwniDt2RgzDHgdKJI1lJM++wF7sl47HxiZ0+0Hcrj/5iYggM9uAuIH/g2MyvY8HahN5l4dQBzQDLgDSLDWplprk4GDQPX8LNTHJgGvAT9kPQ/2nluSeZXUZcD7wCqCv+f9QKGs/7xLAGkEb89fAR2yPc9Jn//Nt2yvzZFADvcL3gTEqWJ8yVp7ylp70hgTCSwm8692mLX2/KlNJ4GS/PF7cH484BhjugNJ1tp12YaDumegNJk7JQ+SeXG9d8i830Ew93yKzCmZL4FYYBpB+j5ba5eQ+cfrvJz0mX3cq94DOdwv6SYggcoYcw2wEXjbWvsu4M62OBI4zh+/B+fHA1E0mZeG3gTUJPNf0TLZlgdjz0eBddbac9ZaC6Tw21/iYOx5MJk9VyXzeNlbZB5vOC8Yez4vJ7/D2ce96j2Qwz2BzPk7/uwmIIHIGFMWWA88ba19I2v406w5Wsich98K7AQaGmOKGGNKAlFkHqgJONbaRtbaxtbaJsD/AV2BuGDuGYgHWhljwowxFYDiwEdB3vOv/G+P9BgQTpD/bGeTkz7/m2/ZXpsjgTyNEcw3ARkBXA6MMsacn3t/EphmjCkMuIDF1toMY8w0Mt/4AsCz1toURyrOG0OA2GDtOeusiEZk/oIXAPoDXxPEPQNTgDeMMVvJ3GMfAewmuHs+75J/no0xs4C3jDHxwDmgc043pk+oiogEoUCelhERkYtQuIuIBCGFu4hIEFK4i4gEIYW7iEgQUriLiAQhhbuISBBSuIuIBKH/B8BadmR5jGGfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr3 = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr3,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#implementing UCB, 1000 runs\n",
    "#CMAB for 2 arms being selected\n",
    "#ln(t) replaced by t\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding the ucb values. we pick the max \n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt(3/2*(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "    \n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    N[arm1_selected] += 1 #will need loop here to do this K times\n",
    "    N[arm2_selected] += 1\n",
    "    reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df.values[t, arm1_selected+1]  #will need loop here to do this K times\n",
    "    rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD0CAYAAABgk2Y8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deYDN9f7H8ecMxtgpScl2U587KinEvdaiyFqoLpGlpjCydVUMkpuKQiTKyNLCrVAYxnIx9tAk4Z4+UtlFFMZyxsyZ8/tjxv1NUXFm+Z7l9fjrnM/3fOf7fs/y8vVdPt8wr9eLiIgEl3CnCxARkZyncBcRCUIKdxGRIKRwFxEJQgp3EZEgpHAXEQlC+Z0uACApKUnXY4qI+KBGjRphlxr3i3AHqFGjhk/ruVwuoqKicrga/6aeQ4N6Dg3Z6TkpKel3l+mwjIhIEFK4i4gEIYW7iEgQUriLiAQhhbuISBBSuIuIBCGFu4hIEFK4i4g4YMfRHVR9qypL9i/Jla+vcBcRyUOedA+Ttkyi/vT6nHCfIKpk7ty05Td3qIqIBLvvf/meLp91Yd2+dTSs2JDpbabj/tGdK9vSnruISC7zer3EJcVRbXI1vj7yNe898B6ruqyicqnKubZN7bmLiOSiQ8mHiF4YzeJvF9O4cmOmtZlGhRIVcn27CncRkVzg9Xr54OsP6LOkDylpKYxvNp7ed/UmPCxvDpgo3EVEctiPp3/kqfinWGAXULd8Xaa3mc5NV9+UpzUo3EVEctDS3UvpOK8jZ1PPMua+MfSt3Zd84fnyvA6Fu4hIDkhJS2H8pvEMXjGYW8vcyr/b/5u/lv6rY/Uo3EVEsinpUBId53Vk1/FdtDat+bDthxSNKOpoTQp3EREfHTt7jL5L+jJr+yyuL3Y9CY8m0PTGpoSFXfLJd3lK4S4icoW8Xi+zd8ym75K+nHCfoF/tfsQ2iKV04dJOl/Y/CncRkSuw98Reei7qScLuBGqXq83U1lO5tcytTpd1EYW7iMhl8KR7GLh8IBM2TSAyfyTjm40nplaMI1fCXI7LCndjTG1glLW2kTGmCjAD8AI7gBhrbboxJhp4CkgDXrLWxhtjCgEfAGWAZKCLtfanXOhDRCTXHDx1kE6fdiJxTyKP3f4YIxqNoGLJik6X9Yf+9FYpY8yzwFQgMnNoLDDEWlsfCAPaGGPKAn2AukBT4BVjTEGgJ7A987PvAUNyvgURkdwTvyueam9XY8vBLUxrPY2ZD8z0+2CHy5s47DugbZb3NYDVma8TgCbAXcB6a22KtfYksBuoBtQDlvzmsyIifm//yf10mNuBVrNbUalkJb586ku63dHN6bIu258elrHWzjXGVMoyFGat9Wa+TgZKAMWBk1k+c6nxC2OX5HK5Lr/qLNxut8/rBir1HBrUs3OWHVjGsC3DSE1PpWOVjgy8fSCeox5cR3O+ttzq2ZcTqulZXhcDTgCnMl//0fiFsUuKivJtwnqXy+XzuoFKPYcG9Zz3zqaeZcDSAbyT9A61rq/F7HazufGqG3N1m9npOSkp6XeX+TI92VZjTKPM1/cDa4HNQH1jTKQxpgQQRcbJ1vVA8998VkTE72w6sIk73rmDd5Le4bm6z7Gu+7pcD/bc5Mue+zNAnDEmAnABc6y1HmPMBDLCOxyItda6jTGTgZnGmHXAeaBjThUuIpITklOSmfzFZGJXxnJ9setZ8dgK7ql8j9NlZdtlhbu1dg9QJ/P1LqDhJT4TB8T9Zuws8FC2qxQRyQUrvl9Bl8+6cDD5IC1uasEHbT+gZGRJp8vKEXrMnoiEnLOpZxm8YjBN3m9C0YiirO22lviO8UET7KA7VEUkhKSlpzHnv3MYuHwgB04doHv17rzZ/E0KFyjsdGk5TuEuIiHhzPkzPPbZY8xzzaPKVVVY1HERzW9q/ucrBiiFu4gEvUW7FtFrcS/2ndzH6Caj6VenHwXyFXC6rFylcBeRoLVm7xoGrRjEhv0bqHpNVdZ2W0u9CvWcLitPKNxFJOi409zErojljU1vUL54ecY3G0+Pmj2IyBfhdGl5RuEuIkFl2XfLiF4Yzb6T+3j8jscZ13QcxQoW+/MVg4zCXUSCQro3nTEbxvDcf54j6pooVnVZRaNKjZwuyzEKdxEJeKv3rGbwysFs2L+B9lXbM/OBmUF5eeOV0E1MIhKw0tLT+Nfqf3H3zLv55tg3vNPyHT5u/3HIBztoz11EApQ9Znky/knW7F3DQ1UfIq5VHCUif3dW8ZCjcBeRgOJJ9zB+03hiV8ZSILwA77Z+l27VuxEWFuZ0aX5F4S4iAePI6SNEL4xm4a6FtLipBVNbT6Vs0bJOl+WXFO4i4vdSPakMWzWMNze/SYonhTfvf5OYWjHaW/8DCncR8WubDmyi/9L+bDywkXZR7Rh5z0hMaeN0WX5P4S4ifumk+yT/XPZPpm6dSsnIknzY9kM63qbn/VwuhbuI+BV7zDL9q+lM2zqN4+eOM/DvAxlUbxClCpVyurSAonAXEb+w/ch2eq/tzZrDawgPC+e+G+/jxUYvcle5u5wuLSAp3EXEcZ/s/IRu87uBF56r+xxP3/U05YqXc7qsgKZwFxHHbPtxGy8kvsACu4Ca19dk1J2juLvG3U6XFRQ0/YCI5LnklGT6L+lPzbiabNi/gWfrPsvqrqspW1jXrOcU7bmLSJ7xpHuYuW0mQ1YO4fDpw3Su1plxTcdxdeGrnS4t6CjcRSRPbD+ync6fdmbbkW3UuaEO8x6ZR50b6jhdVtBSuItIrkpOSWbQikG8teUtShcuzb/b/ZuHb3lYd5fmMoW7iOQKr9fLPNc8+i7py8Hkg/So0YMhDYboKpg8onAXkRy37+Q+esT3IGF3ArdfeztzHp6jQzB5TOEuIjnm2Nlj9F/an1nbZ1EofyHG3jeWp2s/Tf5wRU1e03dcRLIt3ZvO7O2z6b+0P7+4fyGmVgz96vTjL6X+4nRpIcuncDfGFABmApUADxANpAEzAC+wA4ix1qYbY6KBpzKXv2Stjc9+2SLiL46dPcaDHz3Iun3rqHl9TVa0XsFt197mdFkhz9ebmJoD+a21fwdGACOBscAQa219IAxoY4wpC/QB6gJNgVeMMQWzX7aI+IMN+zdQK64WWw5uYWqrqWx6YpOC3U/4Gu67gPzGmHCgOJAK1ABWZy5PAJoAdwHrrbUp1tqTwG6gWvZKFhGnpaWnMTxxOPWn1wdgddfVPH7n44SH6aZ3f+HrMffTZByS+QYoDbQEGlhrvZnLk4ESZAT/ySzrXRgXkQD1zbFv+Mecf7DtyDY6V+vMxOYTKV6wuNNlyW/4Gu79gaXW2kHGmPLASiAiy/JiwAngVObr345fxOVy+VSI2+32ed1ApZ5Dg7/17En3MGv3LCbsmEBEvgjG1BnD/RXu5+D3BznIwRzZhr/1nBdyq2dfw/0XMg7FAPwMFAC2GmMaWWsTgfuBVcBmYKQxJhIoCESRcbL1IlFRUT4V4nK5fF43UKnn0OBPPScdSuKp+KdIOpxE0xubMrnFZCqXqpzj2/GnnvNKdnpOSkr63WW+hvs4YJoxZi0Ze+yDgS+AOGNMBOAC5lhrPcaYCcBaMo7vx1pr3T5uU0Ty2KmUUwxdOZSJWyZSpkgZTR0QQHwKd2vtaeDhSyxqeInPxgFxvmxHRJxxYeqAPkv6cDj5MD1r9mRk45GUjCzpdGlymXQTk4j8ygn3CbrP786n33xK9bLV+fSRT/WouwCkcBcRIOMu0+lbpzNw+UCSzyczuslo+v+tv6YOCFD6qYkI6/atY8DSAWw5tIUGFRvw+r2vU6tcLafLkmxQuIuEsNPnT/PEgif4aOdHlCtWjhltZtD59s66GSkIKNxFQlRcUhwvrn6Rw6cPM7zhcAbWHUjhAoWdLktyiMJdJMT8fO5nnk54mlnbZ1GvQj1mtZtFg4oNnC5LcpjCXSSExO+K58mFT/LT2Z8Y0WgEg+sPJl94PqfLklygcBcJAcu/W86EzROI3xXPbWVuY1HHRdxx3R1OlyW5SOEuEsQOJR+iT0If5rrmcl3R63ih4QsMrj+YiHwRf76yBDSFu0gQ8nq9zLfz6Ta/G+40N8/XfZ6hDYfqhGkIUbiLBJlDyYd4YsETJOxOoHrZ6nzU/iNuvvpmp8uSPKZwFwkSXq+X2Ttm03txb9xpbl6/93V61epFoQKFnC5NHKBwFwkCO47uIHphNJ8f+Jw6N9Rh5gMztbce4hTuIgHsVMopRq0bxWsbXqNkZEmmtppKl+pdNB+MKNxFAlG6N52EbxPondCbPSf20KlaJ8beN5ZrilzjdGniJxTuIgHGHrM8Ou9Rkg4ncWOpG0nskkjDShc9SkFCnMJdJEB4vV7ivoyj/9L+ROaPZFLzSXS/ozsF8xd0ujTxQwp3kQDw05mfiF4YzXw7n3v/ci8zHpjB9cWud7os8WMKdxE/N2v7LPov7c8J9wnG3jeWvnX6akpe+VMKdxE/leJJoW9CXyZsnkDtcrWZ0moK1a6t5nRZEiAU7iJ+xuv18sl/P+HZpc+y9/Renr7racY1HafZG+WKKNxF/MjRM0fptagXc11zMSUNizsu5v6b7ne6LAlACncRP3Dec56Jmycycu1ITp8/zSuNX6FFqRbcdtNtTpcmAUrhLuKw9fvWE70wGtcxF/Ur1Gdyi8ncUuYWXC6X06VJAFO4izjkcPJhRq4dyaQtk6hQogILOyykxU0tCAsLc7o0CQIKd5E8lpaexgdff0CvRb1I8aTQq1YvXm3yKkUjijpdmgQRhbtIHtp5dCcd5nZg+9Ht1KtQj+ltplPlqipOlyVBSOEukgfcaW7GbRzHv9b8i2IFizGr7SweuuUhzd4ouUa/WSK5LHFPIt3md2PPiT20vLklca3iKFu0rNNlSZDzOdyNMYOA1kAEMAlYDcwAvMAOIMZam26MiQaeAtKAl6y18dktWiQQHDt7jNgVsUz5cgo3lrqR/3T+D43/0tjpsiRE+DRBhTGmEfB3oC7QECgPjAWGWGvrA2FAG2NMWaBP5ueaAq8YYzSFnQS1g6cO0m9JPyq9UYmpW6fSv05/NkdvVrBLnvJ1z70psB34FCgODASiydh7B0gA7gM8wHprbQqQYozZDVQDtmSnaBF/teqHVXT+tDOHTx/m0dse5fl6z1P1mqpOlyUhyNdwLw1UBFoClYEFQLi11pu5PBkoQUbwn8yy3oXxi/h6w4bb7Q65mz3Us/855j7Ga9teY+HehZQvUp6Pm3xM1VJV4Ri4jul3+3Kp55zja7gfB76x1p4HrDHGTcahmQuKASeAU5mvfzt+kaioKJ8KcblcPq8bqNSzf1n87WI6L+9MckoyQxsMZVC9QRQqUCjbX9efe84t6vnKJCUl/e4yXyeFXgc0M8aEGWOuB4oAKzKPxQPcD6wFNgP1jTGRxpgSQBQZJ1tFAt651HMMWDqAFrNaUL54ebb12MaIu0fkSLCLZJdPe+7W2nhjTAMywjsciAF+AOKMMRGAC5hjrfUYYyaQEfThQKy11p0zpYs4Z/uR7XSc15EdR3fQs2ZPxtw3RqEufsXnSyGttc9eYviip/Raa+OAOF+3I+JPvF4vr214jaGrhnJ1oatZ1HERzW9q7nRZIhfRTUwil8Hr9fL1ka8ZsmoI8bviaXlzS95u8TblipdzujSRS1K4i/yJ+F3xPLPsGXYd30WRAkUYc98Y+tfpr9kbxa8p3EV+R7o3nYmbJzJg6QCiroliUvNJtK/anmuKXON0aSJ/SuEucgmJexIZtGIQnx/4nOY3Neej9h9pSl4JKAp3kSyOnD5CzOIY5rrmUrpwad574D06VeukQzAScBTuImScMJ25bSYDlw8kOSWZZ//+LMMaDqNIRBGnSxPxicJdQl5KWgo9F/Vk+lfTqV2uNtPaTNN8MBLwFO4S0g4nH6bdx+3YeGAjwxoM44VGLxAe5uuN2yL+Q+EuISktPY2xG8fy6rpXSfGk8MlDn9C+anunyxLJMQp3CTk//PIDjy94nFV7VtG4cmPeaPYGt5a51emyRHKUwl1CRqonlSlJU4hdGUtaehrTWk+j2x3dnC5LJFco3CUkJHybwIBlA/jm2DfcU/keprWeRsWSFZ0uSyTXKNwlqB1KPkSvRb2Yb+dz89U3s+AfC2h5c0tdty5BT+EuQcnr9fLetveIWRxDujedVxu/Sv+/9SciX4TTpYnkCYW7BJ2dR3fSbX43thzaQv0K9Xm39bvcdPVNTpclkqcU7hJUZnw1g75L+lIofyHiWsXRrXo38oXnc7oskTyncJegcDb1LL0X92b6V9OpX6E+7z34HpVKVnK6LBHHKNwl4H195Gs6zevE9qPbGdpgKC80fEF76xLyFO4SsNK96YzdOJbYlbGUiixFwqMJNKvSzOmyRPyCwl0C0g+//EC3+d1YvXc1D/z1Aaa0nKKHaIhkoXCXgOL1eon7Mo4BSweQLzwf09tMp8vtXXTdushvKNwlYBw4dYAnFjzB0u+W0rhyY6a1mUaFEhWcLkvELyncJSAsP7CcYfOHkZqeylvN36JHzR6amlfkDyjcxa/tPbGXl9e+zJQvp1C7XG0+aPsBVa6q4nRZIn5P4S5+6bznPGM2jGHEmhF40j10vqkzUx6eQmT+SKdLEwkICnfxOwnfJvDYZ49x7Owx2kW1Y2zTsZw5dEbBLnIFdNBS/EaqJ5V+S/rRfFZzri1yLYs6LmLOw3N00lTEB9pzF7+w6odV9F3Sl+1Ht9Pnrj6Mvnc0BfMXdLoskYCVrXA3xpQBkoB7gTRgBuAFdgAx1tp0Y0w08FTm8pestfHZqliCyv6T++m/tD9zXXOpVLISnz3yGW3+2sbpskQCns+HZYwxBYB3gHOZQ2OBIdba+kAY0MYYUxboA9QFmgKvGGO0Oyac95xn9PrRRL0VxeJvF/Ovu/+FK8alYBfJIdnZc38deBsYlPm+BrA683UCcB/gAdZba1OAFGPMbqAasCUb25UAt/XwVjrM7YA9bmltWjO+2XjN4CiSw3wKd2NMV+Ana+1SY8yFcA+z1nozXycDJYDiwMksq14Yv4jL5fKlFNxut8/rBqpA7fln98+8/vXrfLbnM64tdC2T60+m4XUNOXf4HK7Df9xPoPacHeo5NORWz77uuXcHvMaYJkB14D2gTJblxYATwKnM178dv0hUVJRPhbhcLp/XDVSB1nOqJ5VpW6cRuzKWE+4T9K7Vm2ENh13RRF+B1nNOUM+hITs9JyUl/e4yn8LdWtvgwmtjTCLQA3jNGNPIWpsI3A+sAjYDI40xkUBBIIqMk60SIjbu38jjCx7HdcxFvQr1eLvF29xS5hanyxIJejl5KeQzQJwxJgJwAXOstR5jzARgLRknb2Otte4c3Kb4qXOp5xi6aihjN46lfInyzP/HfFrd3EqzN4rkkWyHu7W2UZa3DS+xPA6Iy+52JHCs37ee7gu6s+v4LnrU6MHoe0dTrGCxP19RRHKMbmKSHHM29SyxK2IZv2k8FUtWZMVjK7in8j1OlyUSkhTukiPW7l1L9wXd2f3zbmJqxfBqk1cpGlHU6bJEQpbCXbIl3ZvOqHWjiF0ZS6WSlVjVZRWNKjVyuiyRkKdwF5/tOr6LHvE9WLVnFR1u7cCUVlO0ty7iJxTucsXOpZ7jlXWvMGr9KCLzRzKl5RSeuPMJXQkj4kcU7nLZvF4vCbsT6JPQh+9++Y6Ot3VkzH1jKFu0rNOlichvKNzlshw5fYSu87uyZPcSzNVGV8KI+DmFu/whT7qHt7a8xYurX+Rs6lnGNR1Hz5o9Nde6iJ9TuMvv2nl0J48veJxNBzdxT+V7eKPpG9x27W1OlyUil0HhLhfZf3I/4z4fx8TNEykRWYIP235Ih1s76ISpSABRuMuvLNm9hEfmPEJySjJdq3dlVJNRVzR7o4j4B4W7ABk3I43ZMIbYlbHcUuYWPnnoE6pcVcXpskTERwp34Zdzv9Dlsy4s3LWQB//6INPbTKdE5CWfqSIiAULhHuJ2Ht1Jq9mtOHDqABOaTaD3Xb11bF0kCCjcQ9R5z3mGJw7ntQ2vcVWhq1jTbQ11bqjjdFkikkMU7iFo3b51dP2sK9/98h3totrxRrM3uKH4DU6XJSI5KNzpAiTvnEs9x8g1I7lnZsadpYs7LmbOw3MU7CJBSHvuISDdm860rdMYnjicg8kHaXVzK2Y+MJNShUo5XZqI5BKFe5BbvWc1T8Y/ya7ju6hbvi7vP/g+d1e+2+myRCSXKdyD1MFTBxm4fCCzd8zmxlI3MrvdbB655RFdCSMSIhTuQcaT7mHyF5MZsnII5z3nGVRvELH1YykSUcTp0kQkDyncg8iK71cwLHEYG/ZvoF6FekxvM113mYqEKIV7EEj1pPLs8md5c/Ob3FD8Bt5p+Q7Rd0brEIxICFO4B7iVP6xk6KqhbNi/geg7oxnbdKyeYyoiCvdAdd5znmeWPsPELRO5tsi1vNv6Xbrf0d3pskTETyjcA9DP7p/p9UEvEvck0r9Of0bcPUJ76yLyKwr3AJLqSWXSlkkZV8Kkn+f9B9+nU7VOTpclIn5I4R4gEvck0m9JP7Yd2Ua9svWY0nYKUddEOV2WiPgpn8LdGFMAmAZUAgoCLwH/BWYAXmAHEGOtTTfGRANPAWnAS9ba+OyXHTrOnD/DEwuf4N87/k354uWZ3W421cKrKdhF5A/5OnFYJ+C4tbY+cD8wERgLDMkcCwPaGGPKAn2AukBT4BVjTMHslx0ath7eSo0pNfhox0cMbzgc29vyj1v/oUscReRP+XpY5hNgTpb3aUANYHXm+wTgPsADrLfWpgApxpjdQDVgi4/bDQmnUk4xev1oRq0fxTWFr2HFYys0H4yIXBGfwt1aexrAGFOMjJAfArxurfVmfiQZKAEUB05mWfXC+EVcLpcvpeB2u31e1x998dMXPL/peQ6dPUSbim14rvpzlHSX/FWPwdbz5VDPoUE95xyfT6gaY8oDnwKTrLWzjDGjsywuBpwATmW+/u34RaKifDuG7HK5fF7Xn/x87md6LerFRzs/onzx8qzuupoGFRtc8rPB0vOVUM+hQT1fmaSkpN9d5usJ1WuBZUBva+2KzOGtxphG1tpEMo7DrwI2AyONMZFknHiNIuNkq2Sx9fBWOs7ryLfHv+XFRi/yzN+e0URfIpItvu65DwZKAUONMUMzx/oCE4wxEYALmGOt9RhjJgBryTh5G2utdWe36GDhSfcwacsk/rn8n5QuXJr/PPYfGlVq5HRZIhIEfD3m3peMMP+thpf4bBwQ58t2gtnxs8fpMLcDy79fToOKDZj78FxKFy7tdFkiEiR0E5MDVny/gq7zu3L0zFEmt5jMkzWeJDxMj7MVkZyjRMlD7jQ3A5YOoMn7TShSoAjru6+nR80eCnYRyXHac88jX/34FZ3mdWLnTzuJqRXD6HtHU7hAYafLEpEgpXDPZV6vl7EbxzJoxSBKFy5NwqMJNKvSzOmyRCTIKdxz0cFTB4lZHMN8O5+2UW2Z0nIKVxe+2umyRCQEKNxzQbo3nalfTmXg8oGc95zn9XtfZ8DfBmhOGBHJMwr3HLbr+C6eXPgkq/eu5u5KdzOl1RQ9pFpE8pzCPYekelIZs3EMwxOHE5k/kqmtptL9ju7aWxcRRyjcc4A9Znl03qMkHU6iXVQ73rz/Ta4rdp3TZYlICFO4Z8OF69bjvoyjWEQx5jw0h3ZV2zldloiIwt1Xy75bxj+X/ZPtR7fTrXo3Xm78MmWLlnW6LBERQOF+xfac2MPz/3mej3Z+xHVFryO+Qzwtbm7hdFkiIr+icL9MaelpjFwzkpfXvUx4WDjDGw5nUP1BROSLcLo0EZGLKNwvw4+nf6T9x+1Zv389HW/ryKgmo7ih+A1OlyUi8rsU7n/ivz/9l5azWnLkzBE+bPshHW/r6HRJIiJ/SuH+O066TzLu83G8uu5VihcsTmKXRGqVq+V0WSIil0Xh/hter5fpX01nyMohHD59mLZRbXmr+Vu6EkZEAorCPYtUTyo9F/Xk3a3vcle5u5jVbpYeeyciAUnhnmnTgU30WtyLLw9/yZD6Qxhx9whNHSAiASvkw92d5uaZpc8w+YvJXFfsOj556BPaV23vdFkiItkS0uG+7cdtRC+MZsuhLfSr3Y8Rd4+gWMFiTpclIpJtIRnu+07uo+einiz+djHFIorx2SOf0eavbZwuS0Qkx4RUuJ85f4aX177MhM0T8KR7ePmel3myxpN6OpKIBJ2QCffEPYl0/awre0/upY1pwxvN3qBSyUpOlyUikiuCPtw96R5eWvMSI9aMoMpVVVjbbS31KtRzuiwRkVwV1OF+KPkQj857lMQ9iXSu1plJLSZRNKKo02WJiOS6oAz3lLQUZnw1g6GrhnIm9Qwz2sygS/UuTpclIpJngi7cV+9ZTadPO3Hg1AHq3FCHaa2nEXVNlNNliYjkqVwPd2NMODAJuB1IAZ6w1u7O6e2cdJ9k6KqhvLXlLapcVYVlnZbR5C9NdJepiISkvNhzfwCItNb+zRhTBxgD5OhF5fNc84hZHMOR00foVasXLzd+meIFi+fkJkREAkpehHs9YAmAtfZzY0zNnPrCaelpvPTlS8zaPYs7r7uThR0WUvP6HPvyIiIBKzwPtlEcOJnlvccYkyP/qOw8upNZu2fRr3Y/Nj6+UcEuIpIpL/bcTwFZJ2wJt9am/fZDLpfrir9wBBGsbb6Wq4tezXe7vstGiYHF7Xb79P0KZOo5NKjnnJMX4b4eaAV8nHnMffulPhQV5dsVLS6Xy+d1A5V6Dg3qOTRkp+ekpKTfXZYX4f4pcK8xZgMQBnTLg22KiIS0XA93a2060CO3tyMiIv8vL06oiohIHlO4i4gEIYW7iEgQUriLiAQhhbuISBAK83q9TtdAUlKS80WIiASgGjVqXHJ2RL8IdxERyVk6LCMiEoQU7iIiQShgn8SUVw8BcYIxpgAwDagEFAReAv4LzAC8wA4gxlqbboyJBp4C0oCXrLXxTtScU4wxZYAk4F4yeppBEPdsjBkEtAYiyPh9Xk0Q95z5uz2TjN9tDxBNEP+cjTG1gVHW2kbGmCpcZp/GmELAB0AZIBnoYq396Uq2Hch77v97CBVrhbYAAAKRSURBVAjwPBkPAQkWnYDj1tr6wP3ARGAsMCRzLAxoY4wpC/QB6gJNgVeMMQUdqjnbMv/w3wHOZQ4Fdc/GmEbA38nopSFQniDvGWgO5LfW/h0YAYwkSHs2xjwLTAUiM4eupM+ewPbMz74HDLnS7QdyuP/qISBAME3m/gkwNMv7NKAGGXt1AAlAE+AuYL21NsVaexLYDVTLy0Jz2OvA28ChzPfB3nNTMmZJ/RRYCMQT/D3vAvJn/s+7OJBK8Pb8HdA2y/sr6fN/+Zbls1ckkMM91x4C4jRr7WlrbbIxphgwh4x/tcOstRcubUoGSnDx9+DCeMAxxnQFfrLWLs0yHNQ9A6XJ2Cl5iIzJ9T4k43kHwdzzaTIOyXwDxAETCNKfs7V2Lhn/eF1wJX1mHfep90AO98t6CEigMsaUB1YB71trZwHpWRYXA05w8ffgwngg6k7G1NCJQHUy/itaJsvyYOz5OLDUWnveWmsBN7/+Iw7GnvuT0fPNZJwvm0nG+YYLgrHnC67kbzjruE+9B3K4ryfj+B1/9BCQQGSMuRZYBjxnrZ2WObw18xgtZByHXwtsBuobYyKNMSWAKDJO1AQca20Da21Da20j4CvgMSAhmHsG1gHNjDFhxpjrgSLAiiDv+Rf+f4/0Z6AAQf67ncWV9Pm/fMvy2SsSyIcxgvkhIIOBUsBQY8yFY+99gQnGmAjABcyx1nqMMRPI+MGHA7HWWrcjFeeOZ4C4YO0586qIBmT8gYcDMcAPBHHPwDhgmjFmLRl77IOBLwjuni+47N9nY8xkYKYxZh1wHuh4pRvTHaoiIkEokA/LiIjI71C4i4gEIYW7iEgQUriLiAQhhbuISBBSuIuIBCGFu4hIEFK4i4gEof8DNkLRbHgG8b4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr4 = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr4,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAECCAYAAAAFL5eMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3yOV//A8U/2IBERIkRCgktqjy5a9StPtTq0Wh60UaroMFqK2tSmRlGjRhUttaqlFG0p2lKjWipOZCcSWbJ37vv6/XFueUKSIrKd9+vlldzXPOdOfO+Tc53zPWa6rqMoiqJULeblXQBFURSl5KngriiKUgWp4K4oilIFqeCuKIpSBangriiKUgWp4K4oilIFWZZ3AZSqQdM0C2AU0B/5e2UN7AWmCiGySvneDYGLQojqtznuTcBaCLFS07S3ACchxLwSuP904F3gqmmTGeAIfAOMEUKUy3hjTdNcgFghhFl53F8pXyq4KyVlFVAT6CqESNI0rRrwJbAO8C3Xkv3PY8BFACHE6hK+9tdCiOE3XmiaVhP4Gzho+qcoZUoFd+WemVrOrwJuQohkACFEmql13Ml0zEZk6/rjW19rmhYCfAU8ifyAWGA6rz2QA7wghIg0HfeKEOKM6RohwCtAXL6yuAJrAFegLhAK9DFd7wXgP5qmZQC1ARfgO2CREKKl6XwnIBjwAuyBFYAHYAVsE0LMucO3xdV0foLpuj7AJ0AtwAJYJoTYoGnaeWTr/idN0/oBnwM1hRAZmqatA84CPwGfAg6AG3Ae+K8QIlPTtCzgW6C16WfgAcwG0oHT+d6XusAmU50BvhdCTLnDuiiVkOpzV0pCe+CfG4H9BiHENSHErju8hq0Q4hFgKvAZ8IkQojUQDgy8i7L0BX4XQjyKDNDpgK8Q4htkIF8ihPg03/GHgeqapnUwve6HDHwJwGZggxCiPfAQ0E3TtD5F3Pe/mqad1zTtiqZp8cByYJgQ4g9N0yyBncCHpms9AXygadojwG7gGdM1nkZ+GDyuaZoZ0APZtTME+ML0/jQGGgHPms6xBvYKITTTe7UBeNl0n9B85RsCBAkh2gGPA000Tatxh++pUgmp4K6UBCP3/rt040MgELgmhPgr32vnO72IEOIT4DdN00YDK4EWQJF98ab+8A387wNkELDW1K30BDDT1Lo+iWwVtyniUl8LIdoAzU11sUM+cwBoCngDN1rqv5j2t0UG72dMwfxxYDHwH+ARIFAIcQ0YD8RqmjYO2f1V75Y6HTd9fQy4IIS4ZHq9Jt8xPwAva5q2HxiG/KBJKup9USo/1S2jlIRTgI+maQ5CiJQbGzVNq49shb8C6MgHjTdY33KN/A9dc4q4z+2ugaZp85Gt7A3AEWR3yu0eKG4Azpm6QZyEEL9omuZoOq+jECLddG0XIPPfLiSEyNY0bTiyO2UhMBLZDZNkCv43yulq2papaZo1ssvoCvID4WsgF9naB9iK/L+6Hfge+SGTv06p+b7Pvz03X7lOa5rWCOiG7P76Q9O0Z4QQZ2/z3iiVlGq5K/dMCBGJfHi6wRQUMX1dCcQLITKAWKCDaV89ZKv4buW/Rhdk//OtugNLhRCbgRhkK9jCtC8XGexvLf9V4A9kS3edaVsysrU+2nQ/J+BXoOftCimEyAbeBt7RNK0tIIAMTdNeM12rAfLBbnvTKd8A84BDQojLQA1k//nufHX6SAjxten1w/nqlN8xoLmmaa1Nrwfe2KFp2jxgihBiD3JU0z/Iv2qUKkoFd6WkvANcQnaJnEe25i8Bb5r2LwfcNE0TyIeGPxfjHuOBUabr+yJbx7f6CPhY07S/kX3sJ5D91AAHgLc0TZtQyHlrkd0kX+Tb1h94RNO0C6b6bBVCfHknBRVCnEB+4H2K/EukJ/CmqVyHkIH2V9Ph3wDNkP3/mL5GCSHCTa8nAt+YyrEG2a1zo0757xlrKvOXmqadQ/bN37AUaKNp2kXgDPKh8bY7qYtSOZmplL+KoihVj2q5K4qiVEEquCuKolRBKrgriqJUQSq4K4qiVEEquCuKolRBFWIS09mzZ9WQHUVRlGJo3759oZP0KkRwB2jfvv3tDyqEn58fPj4+JVyaik3V+f6g6nx/uJc6nz1b9ARj1S2jKIpSBd1Ry13TtIeB+UKILvm29QdGmLLvoWnaEGRColxglhBin6ZpdsAWoA6QArxumkWnKIqilKLbttxNmejWAbb5trUBBmNKUmTKFT0SmTO7OzBX0zQbZH6NC0KIx5G5pCeXdAUURVGUgu6kWyYQ6HXjhaZptZBJjt7Ld8xDwK9CiCxTGtEAoBUyBekPpmMOIDPSKYqiKKXstt0yQohdppV2bqyTuR54H8jId5gjkD83dAoys13+7Te2FcrPz+9uyp0nMzOz2OdWVqrO9wdV5/tDadX5bkfLtAeaIBcMsAUe0DRtKTLDn0O+4xyARCA53/Yb2wpV3KfF6un6/UHV+f6g6nx3/m20zF0FdyHEH8iVZm6sm7lNCPGeqc99tqZptoAN4IPMV/0rcqmwP5BLiR0v7LqKoihKySqRoZCmpcCWIYP3z8AkIUQmsoXfXNO0E8BQYEZJ3K8shYeHM2LECHx9fenbty/Tp08nNTX1pmP8/PxYsWJFkdc4duwYX3/9dZH7i/Lkk0+SlZV1+wMVRal04g/Ec0o7ReaBf13cq9juqOUuhAhBrulY5DYhxFrkggf5j0kHet9rIctLZmYm77zzDrNmzaJ1a7m4zTfffMOYMWNYs+Z/y1P6+Pj8659VnTt3LvWyKopSORjSDQSODSRyZST2ze2xalVgcbASUWFmqP6bTZtgw4bC96Wne2Bvf/fXfOMNGDDg3485evQoDz74YF5gB3jppZfYunUr48aNIzk5mcTERAYPHsz+/ftZsmQJO3bs4Msvv6RGjRpYWVnRo0cPAIKCgujbty9jxoyhbt26hIeH07JlS2bMmMG1a9eYPn06WVlZJCYm8u6779KtmxpYpChVTcrZFC69eokMkYH7++40mtMI/2D/UrlXpQju5SU8PBwPD48C293d3Tlz5gwDBgxg4MCBnDp1CoDr16+zbt069uzZg7W1NQMK+fQICQlh/fr12NnZ0a1bN2JjYwkKCmLQoEE8/PDDnDt3juXLl6vgrihViDHXSPj8cEKmh2DlakXrH1tTs2vNUr1npQjuAwYU3cr28wsrtafrrq6u/P333wW2h4SE0KFDBxo1anTT9rCwMLy9vbGzswOgbdu2Bc718PCgevXqANSuXZusrCxq167NqlWr2LlzJ2ZmZuTm5hY4T1GUyikjKAM/Xz+Sf0um9n9r03RlU6ycrbhyBSZPhu7d7SmNEKZyy/yLrl278ttvv90U4Hfs2IGzszPm5uaYmd2cjM3Dw4OgoCAyMzMxGo2FfjDceg7AJ598Qs+ePVm4cCEPP/wwal1bRan8dF0nakMUZ1qfIe2fNHy2+PDA1gewrGnFypXQpg0cOgQODsZSuX+laLmXl2rVqrF69WrmzJlDYmIiBoMBTdNYvHgxc+bMKXC8s7MzQ4YMoX///jg5OZGVlYWlpeVtW+JPP/00s2fPZs2aNbi5uZGQkFBaVVIUpQxkRmQSMCKAuD1xOHVxotkXzbD1sCUiQj7vO3wYuneH9eshObl0Rsug63q5/ztz5oxeXJcuXSr2uSUtJydHX7lyZd7r/v3763/88UeJ36ci1bmsqDrfHyp7nY1Gox65LlI/Vv2YftTmqB66IFQ3Gox6To6uL1+u6zVq6Lq9va6vWqXrRqM8517qbIqdhcZV1XIvQZaWlmRkZPDSSy9hZWVFq1at6NChQ3kXS1GUMpAdk40YIoj/Lh6n/3NCW69h18gOPz94/XU4fRq6dIG1a6Fx49IvjwruJWz06NGMHj26vIuhKEoZ0Y06V1dcJXhqMMZMI96LvXEf5Y5RN2PhQpgyBapXh23boE8fKOSxW6lQwV1RFKWYMsMyuTzoMok/J+L8tDPei72p5lMNf38YOBB+/x1efBFWrwZX17ItmwruiqIod0nXdaLWRhHwXgBmFmZo6zTqvlEXXTdj6VKYMAHs7GDLFujfv4jWuq7D0aNY2NoWsvPeqaGQiqIodyEnMQc/Xz/8h/lT4/EadLjQAbfBbgQFmdGlC7z/PnTtChcvwquvFhHYQ0Lg6afhySep/vPPpVJO1XJXFEW5Qwk/J3D59ctkRWXRcEZDPCd5opuZ8emnMG4cWFrC55/LB6iFBnWDAZYvh0mTwNwcVqwgqUsX6pVCWVXLvQgGg4EJEybQt29fXn31VcLCwvL2bdiwgZMnT7Jv3z569+5N3759mTp1KkbjzZMRli9fTvfu3fH19cXX15fnn3+eVatW3XVZli9fztatW++5Trfq06cPERERN23z9fUlMDCwWNc7fPgwY8aMKYmiKUqFYswyEjg2kL+6/oV5NXPa/d6OhlMbEhpuRrduMHw4PP44/POP7GsvNLBfvAidOsmmfZcu8uB335VBvhSo4F6EI0eOALBt2zZGjhzJ3Llz8/adPXuWFi1asHTpUjZt2sS2bdtITU3NOye/gQMHsnnzZjZv3syuXbvYtWsX8fHxZVaPsjJr1iwWLVpU4ANOUSq75DPJnGlzhvCPw6n3Vj06nOuAQwdHPvsMWraEM2fk8MYDB8DdvZALZGXBtGnQrh0EBsKXX8K+fVBI3qqSVCm6Za5tukbUhqhC96Wnp/On/Z93fU23N9yoO6Bukfu7detGly5dAIiMjMTFxQWAlJQU7O3tsbe3Z9u2bXl5ZHJzc7GxsfnXeyYkJOQdl5KSwqRJk/Jmo06ePBlN0+jatSutW7cmLCyMJk2aMHv27LzzDQYDU6dOJSAggJycHDp37sx7771HSEgIkydPJicnB1tbW5YsWUJWVhZTpkwhKysLGxsbZs6ciZubG0uWLOH48ePUrVu3yJmwy5YtIyEhAWtraxYsWMDGjRtxdXXl1VdfJSkpiUGDBrF79+6bzmnXrh3dunUrVt56RamIcpNziVgaQeisUKzrWtNyf0tqPVOL8HAYPFjOMn3ySZmx1tOziIv8/ju8+SZcuiQ74Jcsgdq1y6T8lSK4lxdLS0vGjx/P4cOHWbZsGQDHjx+nU6dOmJub5wX8zZs3k56eTqdOnQpcY+PGjXz//fdERUXh6urKrFmzqF69OgsXLuSRRx6hf//+hISEMGHCBLZu3Up0dDSjRo3C09OTUaNG8eOPP+ZdKyoqijZt2vDaa6/h5eWVF9znz5/P0KFD6dy5M/v37+fSpUvs3LkTX19fnnjiCX7//Xc+/vhjhg0bxunTp9m5cyfp6ek89dRThdb7qaee4tlnn+XLL79kzZo1vPbaa4wePZpXX32Vffv28fzzzxc4p0ePHnnZMRWlsrt+8Dp+A/zIicmh9iu1abqmKZY1rdiwQfaqGAywciUMG1ZEr0pyMkycKA9yd4f9++GZZ8q0DpUiuNcdULfIVnZpr7k4f/58PvjgA/r06cP333/PsWPHGDduHABGo5GFCxcSHBzM8uXLC00KNnDgQPr168fFixcZPXo0DRs2BMDf35+TJ09y4MABAJKTkwFwc3PD09QMaNu2LcHBwXnXcnJy4sKFCxw6dIh69eqRnZ0NQHBwcF4Gyhv54+fMmcOaNWtYt24duq5jZWVFQEAALVq0wNzcnOrVq9O0adNC63xjVm27du345ZdfaNCgAdWqVSMgIIC9e/eycuXKe3pPFaWiMqQbCBwXSOSnkVRrUY2W+1ri+KAjoaEwrD8cPAhPPCFb615eRVzk6FHZ8R4eDiNGwKxZ4OBQxMGlp1IE9/KwZ88eoqOjGTZsGHZ2dpiZmWFmZkZycjLOzs4ATJ06FWtra1auXIn5bR6KtGjRgiFDhjB69Gi2bduGl5cXL7zwAs8//zzx8fHs2LEDgOjoaGJjY6lduzbnzp2jZ8+eXLp0CYDdu3fj4OBAv379sLe3Z/v27ei6jre3NxcuXKBjx4589913JCUl4eXlxRtvvEG7du0IDAzk9OnTNGrUiE2bNmE0GsnMzCQgIKDQsl64cAFXV1fOnDlDkyZNAPnwddWqVbi6uubVX1GqkpQ/U7jUTy6kUX9UfbzmemFua8Hnn8PIkfKY5cvhnXf+pbU+eTKsWCHzC5w4AY8+WqZ1yE8F9yI89dRTTJgwgVdffZXc3FwmTpyIn59f3qpM//zzDzt37qRDhw68/vrrAAwYMID//Oc/RV6zd+/eHDhwgK1bt/LWW28xadIktm/fTmpqKsOHDwfA2tqamTNnEhUVRevWrXnyySfzgvujjz7K6NGjOX78OLVq1cLT05OYmBjGjRvH1KlTWbVqFba2tixcuJAuXbrkre6UmZnJpEmT8PHx4emnn+aVV16hTp061KpVq9By/vjjj3zxxRdUq1aN+fPnA/IZxEcffcTChQtL7D1WlIpAN+pELIkgaEIQVrX/t5BGfDwMfQ1275aDWzZu/Je+9T175JCZyEj5de5cqFatDGtRiKIyipXlv6qSFbIkdOzY8bbHlEed09PT9V69eukGg6HM763rVe/nfCdUnUtfZkSm/mfXP/UjHNEvvHhBz47L1nVd1w8f1vV69XTdykrXFyzQ9SJ/7WNjdf2//9V10PVWrXT95Mm7LkO5ZoXUNO1hYL4QooumaW2A5YAByAIGCCGiNU0bAgwDcoFZQoh9mqbZAVuAOkAK8LoQIrY0PqSU0nPu3DmmTZvGe++9d9vuJ0WpDHRd59rn1wgYHYCeo9N0bVPcBruRmGjG+KFyaKOPjxyxWMiCatKOHXKcemKi7FcfNw6sSmex6+K47f9UTdPGAeuAGwkQPgFGCCG6ALuB8Zqm1QVGAp2A7sBcTdNsgLeBC0KIx4FNwOQSr0EV8+uvv5Z3EQpo164de/fupWvXruVdFEW5Z+kB6fzV7S/EYEH11tXp8FcH6r1Zj19+MaNlS/mw9IMP5Pj1QgN7TAz07i1TPHp4wNmzcsZpBQrscGeTmAKBXvle9xVCnDd9bwlkAg8BvwohsoQQSUAA0Ap4DPjBdOwBQK36rChKuTDmGglbEMaZlmdIOZNC09VNaXOkDVae9kyaJMesV6sGJ0/CwoVgb1/IRb79Fpo3h+++gzlz5MEtW5Z5Xe7EbYO7EGIXkJPvdRSApmkdgeHAEsARSMp3WgpQ45btN7YpiqKUqYzADM53OU/Q+CCcn3HmIb+HqDesHr/+Zka7djJODx4M585BoevrpKbCkCEyf6+7uzxwwgSZTKaCKlbJNE37LzAJeFYIEatpWjKQfyCnA5AI5N9+Y1uh/Pz8ilMUMjMzi31uZaXqfH9Qdb53xlQjaSvSSP8yHTM7MxznO2LxvAV/hoWwcIQru3c74eaWw6efXuP//i+VfCmk8lQ7fpy6s2ZhFRFB/JtvEjt8uBwLWULlLK2f810Hd03TXkM+OO0ihLhu2vwHMFvTNFvABvABLgK/Aj1M+58Bjhd13eJORCrtSUwVkarz/UHVufh0XSfumzgC3gsgKyILt6FueE72xNbdloMHZSv92jUYPx6mTLGiWrUGBS9y/TqMGiWTsjdpAkeP4tK5My73XLqb3Uudz549W+S+uwrumqZZAMuAMGC3pmkAvwghpmmatgwZvM2BSUKITE3TVgFfaJp2AsgG+herBuXAYDAwefJkgoODsbCwYO7cuXiYEv1s2LCBBx54gLi4OL744gssLCxo2rQp06dPv2k0yfLly9m3bx916tQBIDExkR49evD222/fVVmWL1+Oi4sL/fr1K7kKIicmLV68GPd82Y58fX2ZPn063t7ed3yd9PR0xowZQ1JSEnZ2dixcuFBNdFLKTW5yLldGXCF6UzT2ze1p+1tbajxSg6QkGP4mrF8vR8Ls2VNEFwzAN9/A229DfDxMnSpTCdwmd1RFc0fBXQgRAjxielno/1ohxFpg7S3b0oHe91C+cpM/K+SpU6eYO3duXrres2fP0qdPH1588UX27t2LnZ0do0eP5siRIwVGlNxIPwCQnZ1Njx496NOnT5ETiCqj7du307x5c4YPH87u3btZuXIlkyergVFK2Uv4KYHLb1wmKyILz2meNJzSEDMLM378EQYNknOMPvxQJmksdAGk2Fg5CWn7djlU5ocfoE2bMq9HSai4TwPy27RJjk8qhEd6ehGPtW/jjTdgwIAid6uskHeeFXLgwIEYDIYC75WilBVDhoGgcUFcXXEVO82Odr+1w/FhR6KjYexY2LwZmjWTSRofeqiQC+g6fP21zAWTnFwhx63frcoR3MuJygp551khLSwsGDBgAP7+/nz++ecl8fYryh1JvZiKXz8/0i6m4f6eO43mNEK3smDJEpg9G1JS5MCWKVPkuqYFJCXJLpitW2Xk37BBDnes5CpHcB8woMhWdpjKCllhskJu2rSJwMBAhg0bdtOHkqKUBt2oc3XFVYLGB2HhaEHLAy2p9XQtLl+W4eL0aZnBceVKeOCBQi6QkwPLlsk8MImJMHOm7LOpwMMb70bVqEUpUFkh7zwr5Jo1a3B1deXFF1/E3t4eCwuLe3rvFeV2MsMyuTzwMolHEnHu4UyzDc2wrG3N0qWylV6tmuxl6dOnkJN1XeZXHzMGhIDu3WUTv337Mq9HaVLBvQgqK+SdZ4V8+eWXGT9+PLt27cJgMDBnzpziv/GK8i90XSd6UzRXRl4BI3k5YcLCzBjYVaZSf+45mRumbmFLQFy+LIc3HjoETZvC3r3w7LNFLHpayRWVUaws/6mskP+jskIWrqr9nO+EqvPNsmKy9AsvXdCPcEQ/99g5PT0wXTcadX3TJl13dNR1BwddX79e143GQk7Oztb1OXN03dpa152cdH3JEl3Pyiq9ityF0soKqVL8Kbd17tw5+vTpwzvvvKOyQirlIvF4ImdanyH++3i8FnjR5mgb0hzt6NNH9q+3agV//SUHwRVohO/bJw+YOBFeeEG23t97D6yty6UuZUV1y1QwFTkrpKKUNWOWkeDJwYQvCceukR2tDrSieuvq/Pwz+PpCXBzMmyezOBZ41BMcLBc8/fZb2QXz7bcyuN8nVHBXFKVCSv0rFT9fP9IupOE2xA3vhd7kWFsydiwsWiTj9fffFzHH6Ouv4c03wWiE+fNlkK/EY9aLQwV3RVEqFGOOkfCF4YRMD8HS2ZIWe1vg8pwLJ0/KdaeFgGHDZIAvsJJdSooczrhypVy/dNs2mXP9PqQ6UBVFqTByLuVw7pFzBE8KxuVFFx68+CDVurowdix06gQZGXD4MKxefUtgNxplQPf2ll/HjIFffqnQgT3XaGRjVBRXTbO7S5oK7oqilLvc1FwCRgdwvc91sq5m0XxXc5pvb84fl61p2xY+/lj2sly4AN1uXfInKkqOVX/3XTmz9ORJeUIF7YbRdZ0TiYl0Pn+eQULwd07O7U8qBhXci7B7924+/vjjfz0mJCSERYsWAfD111+Tk5NDZmYm48ePR9f1Ase3aNECX19ffH196du3L3369CE8PPyuyzZw4MC7Pud2jh07xocffnjTtlOnTvH+++/f9bUOHz5MdHR0SRVNqeKSzyRztu1ZIpZEYNfbjocuP0S1p2vz3nvQuTNkZclh6WvWgKPjLSfv3QutW8Ovv8rB7T//DA8/XC71uBOnk5N55Nw5Hj9/nktpaWz18eGZQjOY3TsV3O/B/PnzGTRoECBnaRqNRmxtbWnbti179uwpcHyNGjXYvHkzmzdvZtu2bfTq1atK5mHZtGkTqamp5V0MpYIzZhkJmRXCnx3/xJhppM3RNjhOc+TkBStatYJPPpGN8QsXoMDcwIQEuY7pCy+Am5tcx/TNNyvsZKTwzEx8/fx46Nw5wrKyWN20KeGPPkpfV9dSu2eleKC66do1NkRFFbovPT0d+z//vOtrvuHmxoBCp7DdLCIigjFjxlC3bl3Cw8Np2bIlM2bMICgoCF3XcXZ2ZseOHcTGxvL++++zcuVKnnnmGd58801eeumlf712ZGQkjqamyIEDB9i4cSPm5ua0b9+eDz74gOXLlxMUFER8fDzJyclMnjw5L+8LwB9//MGKFSsAuZrL/PnzadSoEStXruTHH3/EYDDQr18/+vbty+bNm9m3bx9mZmb06NGDAQMGEBgYyMSJE7Gzs8POzo4aNQqughgaGsrgwYNJSEigX79+PP3007z00kscPHgQCwsLFi5cSIsWLXjmmWcAOHr0KH5+fowfP56vvvoK6yo+llgpnus/XufKO1fIuJJB7T61abqqKekWVsx6x8jWrdCwIRw5AqbErDc7fhxee03m750zR/avV9Dfs3SDgeVXrzIjJASjrjPRw4PxHh44lkH+mkoR3MtbSEgI69evx87Ojm7duhEbG8vp06cxLVZC7969WbVqFUuWLAFkCz0hIYGUlBQcHP63+mBSUhK+vr6kpqaSmJjIU089xciRI0lMTGT58uXs2rULOzs7xo4dmzfe3dbWlk2bNnHlyhXGjBnDd999l3e9K1eusHDhQlxdXVm9ejU//PADTzzxBMeOHWPHjh1kZ2ezaNEirly5wv79+/nqq68wMzNj4MCBPPbYY3zyySeMHDmSTp068dlnnxEUFFSg7jk5OaxatQqj0UjPnj3p2rUr7du358SJEzz22GMcO3aMUaNG5R3fpUsXfHx8mD59ugrsSgHZ0dkEjA4g5qsY7Brb0epgK5y6OfPNNzByJERF1WT4cBmzq1e/5eRr12TimI0boVEj2RVTaP7e8pduMLA6MpIFYWFE5+TwQq1aLGvSBM9S6oIpTKUI7gPq1i2ylV0WS5F5eHhQ3fSbVrt2bbKyskhISPjXBTdcXFxITEy8Kbjf6JYxGAx8+OGHWFlZUa1aNf7++2+uX7/O0KFDAUhLS8vri3/kEblGSpMmTYiLi7vpHq6ursyePRt7e3uio6Np164dwcHBtGrVCgsLC+zs7Jg8eTL79+8nMjIyr68+KSmJsLAwrly5QqtWrQA5Uamw4N6mTZu8IO3t7U1ERAS9e/dm8+bNGI1GOnbsqIK4clu6rhO9JZqA9wIwpBrwnOaJx4ceBIVb0Ps/squ8VStYsiSEPn0a3XqyHNI4apRcGenDD+VsUweHwm9WjnRdZ7QKX/IAACAASURBVEdsLO8FBBCVnc2TTk5sb9iQzk5OZV4W1ed+BwpL5VurVq28NL03jjEajXmv82ePvJWFhQUzZ87k8OHDHD16FHd3d9zc3NiwYQObN2/mtddeuylBGcgUwa639M9NnjyZOXPmMG/ePOrUqYOu63h5eXHp0iWMRiM5OTkMGjQILy8vGjduzKZNm9i8eTO9evWiadOmeHl58aepS+vixYuFlvXSpUvk5uaSnp5OYGAgHh4edOjQgfDwcHbu3Mkrr7xS6PtV2ANl5f6UGZbJhWcvcHnAZeyb2dPhrw7Um9CI2QssaNkSzpyB5cvl15YtM28++fp16NsX+veHBg3gzz9lit4KGNiPJCTwn7/+4r+XLuFmbc2xNm34qU2bcgnsUEla7hXRQw89dNMqSR06dGDo0KFs2rSJlJQUHB0dqVZghsX/2NraMnv2bMaPH8/evXsZOHAgvr6+GAwG6tevn9eH7efnx+uvv05GRgYzZ8686Ro9e/akT58+ODo64uLiQkxMDD4+Pjz++OP069cPo9FIv379aNasGY8++ij9+vUjOzubVq1a4erqyrRp03j//fdZv349zs7Oha4kZWNjw5AhQ0hOTmbEiBE4mX5Rn3/+eX744Ye8lMD5tW3blnHjxrFhw4a845X7j27UiVwdSdB4+Xyq8bLG1H+3Pr8cM+OtF+VkpP/+FxYvhnr1CrnAwYNybbzYWNlPM25cITkGyt/JpCTmhIWxNz4eVysrljZuzLv16mFZ3nmYisooVpb/KmtWyGHDhumxsbEFtm/ZskXfs2fPPV9/2bJl+ldffVVge0XIFvjZZ5/pO3bsKLP7VYQ6l7XKXOe0y2n6uc7n9CMc0c//57yeHpyux8To+oABug663qiRrh84UPC8S5cu6fo//+j6Sy/JA318dP3s2bKvwB34LTFR737+vM6RI7rz8eP6rJAQPSM3966vo7JCVkBjx44tMJQxMzOTc+fOFboUXVXx4YcfcubMGV64j5IwKXfGmGMkeGowfzT/g9S/UtHWa7Q62IrvTtnRrJlcyW7iRLh4EZ5++paT09JwnTMHWraEH3+EGTPkEMd27cqlLoUx6jpHExLo/tdfdPzzT86mpjLfy4vQRx5hkqcnthXoL4s76pbRNO1hYL4QooumaY2BjYAOXATeFUIYNU0bAgwDcoFZQoh9mqbZAVuAOkAK8LoQIrYU6lEuvL29GTt27E3bbG1t8yY23asRI0aUyHVK2rx588q7CEoFlC7S8RvgR8ofKbi+7or3fG+u69b07g27dsm5RevXF7E86fffw1tv4RwRAe+8Ax99BP8yYKE8nE9J4S1/f06lpOBiZcUCLy/erleP6hV0Wb7bttw1TRsHrANujOFZDEwWQjwOmAE9NU2rC4wEOgHdgbmaptkAbwMXTMduAiaXfBUURSlPukEn4pMIzrQ9Q0ZABg9sf4Bmn/vw1QFrfHxkOvW5c+HEiUICe2SkXAvvuefAyYmQLVvg008rVGCPyc5mqBC0O3uWwMxMVjZpQsgjjzDWw6PCBna4s5Z7INAL2Gx63R74xfT9AeApwAD8KoTIArI0TQsAWgGPAQvyHTulhMqtKEoFkPZPGuJNQfLJZJx7OKOt0whLteG17jLB12OPydZ6gbXYDQZYtQomTZL5BWbNgrFjyQgMLJd6FCYxJ4elEREsu3qVFIOBUe7uTPX0pGYFzVlzq9sGdyHELk3TGubbZCaEuDHOLQWoATgCSfmOKWz7jW2F8vPzu/NS55OZmVnscysrVef7Q0Wus56tk7Y2jbQ1aZhVN8NxniPZXa0YOT2Fzz+3wsZGZ/LkWPr2TcBggPzVsBECt6lTsbtwgdSOHbk2ZQo5np4QGFgh6pyj62zPyODT1FQSdZ2HrayY7OyMd04O1wICuFbC9yutOhfnbwpjvu8dgEQg2fT9v22/sa1QxZ2IVBaTmCoaVef7Q0Wtc+qFVPxe8yPtYhp1+tbB+5PG7P/VmvdehrAwuULSggVQt25dIN/kw/R0mSxm40ZwcYEtW6jevz+N880jKc8667rO3vh4xgcFcTk9nf9zcmKxtzdtSnlM/b3U+ezZs0XuK05w/1PTtC5CiKPAM8AR4A9gtqZptoAN4IN82Por0MO0/xngeDHupyhKBWDMNhK5OpLAcYFYOlnS4rsWJDRzoecAOSS9ZUs4dgwef7yQk3/4QT4oDQmR49XHjpUBvoK4lJbGO/7+/JKUhGZnx3ctWvBcrVqFTmCsLIoT3McAazVNswb8gJ1CCIOmacuQwdscmCSEyNQ0bRXwhaZpJ4BsoH9JFVxRlLKT9HsSYrAg3S8d5x7OeK1pxqL11sx5BWxtYelS2Sgv8HwxKkouRr19O2iazAb2xBPlUofC/Hj9OvPDw/kxIYGalpasatKEN93cyn8CUgm4o+AuhAgBHjF97w8U+OkIIdYCa2/Zlg70vudSKopSLnSjTtjcMIKnBGPTwIYW37Yg0LUWjz5jxsWL0K+fnGFaIPWTwSCXS5o4UT4w/egj2WIvZBZ0ebiYmsqcsDC2xsTgbmPDzIYNGVqvHnWqUJ6kijuOR1GUcpUZnokYIkg4mECd/nWo/3FTFqywZN48mUJ97145grEAIWTagN9/h65d5aiYQtJUlIdzKSlMCQ7mwPXrWJqZMcXTk4keHhVq8lFJUcFdUZSb6LpO9KZoroy8gm7QabKyCefq1+OJVmbExcm4vWQJFEj/bzDAsmWytW5nB5s2ybzrFaDfOiori/lhYayKjKSGpSVjGjRgtLs7bhXkL4nSoIK7oih5smOy8R/mT9yeOGo8VoNGa5oxZZUdK96BNm3g22+hY8dCTjx5EoYPl+kCnn9eronn5lbm5b/VhdRU5oaFsTM2FqOuM6BuXRZ4eeFShbpfiqKCu6IoAMR+E4v/MH9yk3LxWuhFSvcGdO5rxoUL8pnovHmFdJmnpMCUKTJnb/36sGWLTM9bzq3108nJzA4N5dv4eKpbWPBOvXqMcHfH286uXMtVllRwV5T7XHZcNgEjAojZFkP1ttVp9WNrNh2vztiH5WpI338PPXrccpKuw549MGKETCHw1lsy+hdYwbpsRWZlMeLKFXbHxeFkack0T09GurvjXElmlZYkFdwV5T4W/0M8YpAgJz6HhtMbYj3Qg1eGmnPoEHTvLucbFRgJ888/MHo0HDokl0/auRNMK4aVl+jsbNZFRbEgLIxsXWdmw4aMdHcvk7VKK6r7t+aKch8zpBkIHBdI5MpIqrWoRrPvWrF8f3UWNgejUQ5wGTbslt6VnByYPv1/LfSlS+XEpHJsFWcaDKyJimJiUBDpRiNP1azJp02a0NjevtzKVFGo4K4o95nkU8n4+fqREZCB+xh3Yp5rRNehFpw/Dy+/LHN4NWt2y0lnz8Ibb8Dff8vhMgsXlmvmRqOuszUmhklBQYRmZdG9Zk0WenvTssCq2vcvFdwV5T5hzDUSOiuU0Fmh2NSzocl3rZlzoCYr/w/q1JEjYQqsv5KRIRfN+PhjedCePdCzZ7mU/4YjCQmMDQzkbGoq7apXZ42m8VTNmpU6VUBpUMFdUe4D6QHpXPa9TPLJZFx9XTn/eGP6vG3F1atyJMzMmfLh6U1OnIDBg8HfX379+GMoxzVxTycnMzE4mB8TEmhgY8PmZs3o7+qKuQrqhVLBXVGqMF3XifosioDRAZhbm+Ox1ofxB13ZORQ6dIAdOwp5FpqSIiciffopeHrKxOzdupVb+Y8kJrLi6lW+iYujtpUVi729eatePeyq4KzSkqSCu6JUURlBGfi/7U/CoQRq/qcmyW9pPDnalshImDNHpnopEB8PHYIhQyA8HEaOlB3w5dCPnWs0ciQxkekhIfyWnEwNCwumeXoypkEDHO7jETB3Q71LilLFGHOMRCyOIGRGCGaWZjRa2oSVV+ux8BUzGjaUvS0PPXTLSQkJcnjjxo3yaeqJE0VMRS3Fcus6v2ZlseDyZfbHxxOTk0N9a2tWNWnCa66uFXpJu4pIvVuKUoUkHE0gYFQAaX+n4fKSCykDm/DsRBv++Uc2yBctggJrT+zeLfP1xsbK7pgpU2Qe3zIk0tN59dIlzqam4mBhQQ9nZ15wceElFxfV/VJMKrgrShVgzDUSMiWEsHlh2Hja0Hhrcz4+WZtlL8qsAIXOMo2Olvlgdu6UiWP274e2bcu03CcSE1kYHs7B69epbmHBHEdHhrdqpbpeSoB6BxWlkks+lczlwZdJ/ycdtzfdON+5Mb6TLQgMlHOM5s27pbWenS3TOs6dK4c6zp4tV0Yqw8lIJ5OS+Cg0lAPXr+NqZcVb9eox1sOD5KAgFdhLiHoXFaWSMmQYCJ0dSvj8cKzrW+P6WXNG7qzNoQHQogX89BM8+eQtJ507Jych/f03PPusHN5YYMZS6dB1nYPXrzMvLIxfkpJwsbJiTqNGjHJ3x97U9ZJcJiW5P6jgriiVUMLPCfi/7U+GfwZ1XnPl9MONGfmBFQaDHMH41ltw00pxaWlyNaTFi6F27SJmLJUOo67zTVwcc0JDOZeairuNDXMbNWJ4/frqIWkpUu+solQixlwjYbPDCJkRgl1jO+pvbcX7W5z5foRcmnTDBvDyyneCrssRMOPHywemgwbJ1rqzc6mXNcdo5KuYGOaFhXE5PZ0mdnas1zRec3XFugqsUVrRqeCuKJVE2j9pXB54mZQzKbj6unKuc1OGv21BVhZ88ol8NnpTzLyRinfvXnj8cZk6oAyGN6bk5rIpOpoFYWGEZWXRulo1vn7gAV6uXRsLNZu0zKjgrigVnDHXSMSiCIKnBmPpaEn9zx5g7L7afDvEjI4dZcP8piVKdR1WrpStdYNBPjwdOfKWyF/y4nNyWHH1Kp9ERJCQm8uDDg6satqUZ5ydVd6XclCs4K5pmhXwBdAQMABDgFxgI6ADF4F3hRBGTdOGAMNM+2cJIfbde7EV5f6QLtLxe92PlFMpuLzswqWnmtJzgjWpqXLM+qhRt8wyPXhQBnJ/f3j6adkBf1M/TclLzc1lcnAwn0VFkWE00rNWLcZ5ePCoo6MK6uWouB/lPQBLIURH4CNgNrAYmCyEeBwwA3pqmlYXGAl0AroDczVNq7or0ipKCTHmGEnbkMaZNmfIuJKB+xofZtCc/w6zxtsbzp+XE0rzAvv169CnjwzoRqNcnHr//lIN7Lqus/naNXxOn2bZ1av0rVOHiw8+yJ6WLelYo4YK7OWsuN0y/oClpmnmgCOQAzwC/GLafwB4Ctmq/1UIkQVkaZoWALQCTt9TqRWlCkv7Jw2/1/xIPZ9KredrceWFpvSaaENSkhyzPmYM5A0y0XX46it4/32ZQmDWLPjgg0IWOy1ZJxIT+TAoiF+Tk3nQwYEvfXzoXI4ZI5WCihvcU5FdMpcBF+A5oLMQQjftTwFqIAN/Ur7zbmwvwM/Pr1gFyczMLPa5lZWqc9Wk5+qkb0ondVkqZtXN0GdUZ8KpBnw/xIbmzTNYty6SJk2yuXJFHm+RmEjdGTNwPHiQ9NatubZ6NVk+PhAUVGpljDAYWJ6ayt7MTJzNzJjp6MhLtraYR0XhFxV1z9e/H37OtyqtOhc3uL8PHBRCTNA0rQHwM2Cdb78DkIick+BQyPYCfHx8ilUQPz+/Yp9bWak6Vz0ZIRn49fMj9WQqtXrWIqSXxqDR5iQnWzJzJowfb4eVlff/TjhwQOZYj4uDuXOxHzsWr1LMwRKYkcHs0FA2xcVhYWbGJA8PJnh6Uq2E71nVf86FuZc6nz17tsh9xQ3uCciuGIDrgBXwp6ZpXYQQR4FngCPAH8BsTdNsARvAB/mwVVEUZL/1tQ3XCHg/AMzAY60P03+pw+bXzdC0TH76yZLWrfOdkJoqUwWsXg3Nm8t+9TZtSq18/unpzA4N5cvoaKzMzRlevz5jPTyoX8rdPsq9K25wXwJs0DTtOLLFPhE4A6zVNM0a8AN2CiEMmqYtA44jH95OEkJklkC5FaXSy4rMQgwRXN9/HacuTkQN0Hh8sh3R0TB1KvTqFUzr1vladL//DgMGQGCg7HifNavUsjf6p6fzUUgIW2NisDE3Z5S7O2MbNKCuCuqVRrGCuxAiFehTyK4nCjl2LbC2OPdRlKpI13VitsZwZfgVjJlG3Oc3Zu7l+mx4w4wWLeSco3btIK8bNiUFJk2CFSvAwwOOHJHTUUtBSEYGC8PDWRsVhbWZGR80aMCYBg2oY219+5OVCkVNYlKUMpQdk43/O/7E7YrD8RFHEt9qRtep9kREwIQJMG3aLQNdvv1WTj29elWmeJwzBxwdS7xc51NSWBwRwdaYGMyB11xdmeflpYJ6JaaCu6KUAV3Xid4cTcB7ARjSDDSY5cVniQ1YMNCMpk3ht9/g4YfznXD1KvVHjoQff5QpHgtd7PTeXU5LY1RAAIcSEqhuYcE79eoxtkED3Mt4sQ6l5KngriilLCMoA/93/Ek4mIBjJ0cS3tDoNrMaISEwdKjMDmBvbzrYYJAPSydMoHp2tsy5PmZMiedav5KezpjAQPbGx+NgYcECLy+GuLnhVIY53ZXSpYK7opQSXdeJWhdF4OhAMAfnqY1ZFlaf9YPN0DT4+Wf4v//Ld8Lff8tof+oU/Oc/BI0ZQ+Pu3Uu0TFFZWXwUGsq6qChszc2Z2bAhg93ccFMPSqscFdwVpRSkX0nHf6g/iUcTcXrSiaMPNWPcXFvMzGTagJkz87XWMzJkrvWPPwYnJ9i8GV59lZzLl0usPH+mpLA2Koovrl0jW9cZ6ubGFE9PNfqlClPBXVFKkG7QCV8STsiUEMxszHCe05TRP7rx0zwzXn5ZdsE0aJDvhMOHZVreoCCZa33hQqhVq8TK81dqKtNDQtgTF4eduTkvubgwo2FDGud9sihVlQruilJCUi+mIt4QpJxOwenZWnzftClzZ9tgbg6rVsGwYZCXSys2Vjbht2yR+XoL9NHcm79TU5kREsLuuDhqWFgwvWFDRtWvr/rU7yMquCvKPTJmGwmbF0borFAsa1hiNdOHflvqcPl7M3r3lo1xT0/TwTdWRvrgAzl+fcoUmDixxCYjXUxNZUZoKDtjY3G0sGCqpyfvu7uroH4fUsFdUe5B8plkxBuCtAtp1OpTh51ujZk1zZr69WVq9aeeynewv7/sgjlyBDp1gs8+gwceKJFyXEpLY0ZICDtiY6luYcFkU1B3VkH9vqWCu6IUgyHDQMj0EMI/Dsfa1RrzuS3ou9mFS9tlPq9Fi6DGjfynWVmy+X4jXcDq1TBkSImsjHQxNZXZYWF8HRNDNQsLJnh4MLpBA2qpoH7fU8FdUe5S4olExGBBhn8Gzr51+czcm08nWOHpKZM1Pv206UBdh1275HJ3QUFyMY2lS8HN7Z7ur+s638fHsyYqin3x8VQzN2e8hwdj3N1xUTNKFRMV3BXlDuWm5hI8IZirn17F1tOWhEmt6LvOmdhYOc9oxgyoVs10cEAAjBgBP/wgZ5gW6KMpnvMpKYwODORIYiIuVlZM9fRkpLu7aqkrBajgrih34Prh6/gP9SczNBOHAfWZGdWIb2db0r69zLrbrp3pwJQUmDxZrl1qYwPLlsmcMPeY9zwqK4vJwcF8fu0azpaWfNqkCUPd3LAs5UWvlcpLBXdF+Rc5iTkEjgnk2oZr2Gl2iHfbMnpDDczM5Jj14cPzLXl36JB8YBoSIsc9TpsGdeve0/3TDQYmBwezKjISg64zpkEDJnl4qNEvym2p4K4oRbh+6DpisCArKgvbwR5MFJ78vMKC556TDXMPD9OBMTFyDdOvvoKmTeHYMXjssXu6d5bRyJ6MDL48e5ZL6ekMrFuXSZ6eeNvZ3XvFlPuCCu6KcovsmGyCJwUTtS4Ku2b2/DGgHRM+d8TBQQ5RHzDANBlJ12HDBrkyUmqqXGFjwoR7GrOeZjCwOjKSj8PDuZadTTN7e/a1bEmPEpy1qtwfVHBXFBNd14nZJhfRMCQbsB7QgOHnG3LmcwsGDZKpX5ydTQdfviy7Xo4dg8cfhzVr4B7W/kzNzeVTU1CPy8nhSScnZtnb80br1pjlTWtVlDungruiAFnXsrjy9hXi9sRRrYMDv3ZqxriV1XBxkaMZe/W6cWCWTMM7d67M/LV2LbzxRrHHrCfn5rLi6lUWh4cTn5tL95o1mdKwIZ1q1MDPz08FdqXYVHBX7ms3tdbTDGQM8KL/wQZEfSITfa1ZY8rjpeuwe7dMFeDvD/36ySeqrq7Fum9iTg7Lrl5lSUQEibm5POvszJSGDXm4FFZZUu5PKrgr9638rXXr1g587tKMDZuq0bo1bN8pMwSYmQFhYbJ1/tNP8oHpTTOV7s71nBw+iYjgk4gIkgwGetaqxZSGDWnv4FCylVPueyq4K/elmB0x+L/ljyHNgF9nL97/rQGWNmbMni2fj1pZIVvr69bLGUpGI3zyCbz9drFWRYrLzmbF1assu3qVhNxcerm4MMXTkzYqqCulpNjBXdO0CcALgDWwEvgF2AjowEXgXSGEUdO0IcAwIBeYJYTYd6+FVpTiyo7L5srbV4jdGYuhqQNjzZpx/ng1Bg2SqV/yMgPcWAPv8GHo0gXWrwcvr7u+3/WcHBaFh7MoPJxsXedpZ2fmeXnRqnr1kqyWohRQrOCuaVoXoCPQCbAHPgAWA5OFEEc1TVsN9NQ07XdgJNABsAVOaJp2WAiRVRKFV5S7kfBTAn6+fuTE5/DXg40YfboBLVqZc+ZQvhmmcXFyVaQ1a8DaWiZiHzr0rh6Y6rrO6ZQUvrh2jS+jo0kyGOjl4sLMRo14IC8/gaKUruK23LsDF4BvAEdgLDAE2XoHOAA8BRiAX03BPEvTtACgFXD6XgqtKHfDkGEgZGoI4YvCMfOwY4ZrS46ecWDsOBnHbWyQXTDbt8t8MNevy1WRpk69Zdmk2/NPT2f4lSscTkjA1tycXi4ujPfwUC11pcwVN7i7AJ7Ac0Aj4DvAXAihm/anADWQgT8p33k3thfg5+dXrIJkZmYW+9zKStX5zui6TvbRbFIWpGAINRDcvCYj/Jrj6KqzcWMoDz6YTlAQWF67Rt2ZM3E4coSMli2J+uwzsjRNTky6w3tGGQysS0tjZ0YGZsD46tXpZWeHg5kZhIdTnJ+W+jnfH0qrzsUN7vHAZSFENiA0TcsE8jdxHIBEINn0/a3bC/Ap5gQQPz+/Yp9bWak6315uaq5coHprIrqbLZ82asGuf2rSrx+sXAlOTp6ytb7e9MA0JwcWLcJu1Ci87iLJV2JODlNDQlgTE4MRGOTmxkQPDxqWQJoA9XO+P9xLnc+ePVvkvuIG9xPAKE3TFgNuQDXgJ03TugghjgLPAEeAP4DZmqbZAjaAD/Jhq6KUmsRfEhFDBRkBGfzWrCHTLnvg3tD85slIYWFywYxDh+QD03XrwNv7ju+RZTSy6upVPgoNJSk3lzfd3Jjo6YlnCS2Xpyj3qljBXQixT9O0zsjgbQ68CwQDazVNswb8gJ1CCIOmacuA46bjJgkhMkum6IpyM2OWkZCPQgibG4bR1ZY5Tq05HlSTOQtg5Mh8fevr18vFqY1GmQHsrbfu6oHp7thYJgYFITIy6OrkxMfe3mpIo1LhFHsopBBiXCGbnyjkuLXA2uLeR1HuRMq5FC4PvEzahTT8vesyKrAx3s0tObUF2rQxHRQaKke+3Gitb9gAjRrd0fWv5+Twc0ICn0VFcTghgaZ2dnzXogXP1aqlUgQoFZKaxKRUasZsI6EzQwmdG4pew5rlri34LsSFKdNlpgArK2R/+pIlMH26bKHfRWs9zWBgbGAgayIjMQJ1rKxY4OXFe+7uWKmFMpQKTAV3pdLK31oP9Hbl/cDG1Pay4thu6NjRdNCpU7K1/vff0LOnXBkpLxF70dINBtZGRbEgLIzI7GzerVeP/q6uPOjgoIK6Uimo4K5UOjkJOYTMCOHqiqsYHa1Z5NSCgyEujJsgh6bb2iKzN06bBgsXymmn33wDL75422sn5+ayKjKSReHhxObk0LlGDbY3b06nGoWO4FWUCksFd6XSuJHBMXBMINkx2fh5ujEuyAvv1lb8sSHfLNNz5+D11+HiRRg8GBYvhttkW8w0GJgXFpaX+6V7zZpM8vTkcSen0q+YopQCFdyVSsGYaOSfXv8QtyeOnEbVmezQkj/DHZg6E8aPN/WtBwfLlZB27JCpePftg2efve21TyQmMlgI/DMyeNHFhYkeHjyoUu8qlZwK7kqFF/dtHPHD4jHG6xxu7M28AHc6PGTGnxugeXMgNxfmL5J9MhYW8M47MGNGvmWTCnchNZUlERFsvHYNT1tbDrZqxVO3OUdRKgsV3JUKKysyiysjrhC3O46EmjZMNDQnKc2RxUth+HAZxzl9Wj4wPX8eXnkFli6F+vX/9br+6enMCAlha0wMFmZmjHJ3Z2bDhlS3VP8dlKpD/TYrFY5u1In8LJKg8UHkZhjZVbMRqxMaMHCwOUuWgIMDkJICkyfDihVQt65cC++ll0yraxQuKCODj0JC2Bwdja25OeMaNGCshwe1ipGfXVEqOhXclQolzS8N/6H+JJ1IIsrNibHJTbF3sWfD0lAGDPCUB+3ZI5vukZGyC2b2bPiX0SwXU1OZHx7O1uhorMzNec/dnXEeHrhaW5dRrRSl7KngrlQIukEnbGEYIdNCMFpbsNpRY3dMXcZOMGPKFAgJSYeICJmSd88eaNkSdu6ERx4p8poB6elMCg5me2ws1czNGenuzgcNGlDPxqYMa6Yo5UMFd6XcJZ5IJPD9QFLOpBDsUZvRYU3wbG3NmS+gdWvAYKDmli2wfLl8eDpvnswNU0R3ytWsLCYEBfFldDT2FhZM8PDggwYNcFbdL8p9RAV3pdwYs42ETAshbH4YubVsWGnfjP9v784Doir3P46/hx0UEQIRlEVRHrdcrtrifrtl2XpbrN8vLdNc6mqaWVauaXjN1DQrKy2XNG+LWTctRCl3bwAAFfZJREFUTc0000yl9CcyPIiIoKKisgozMMvvj8Gi1HsTgWGG7+uvOWfmzHm+zPDhcM5znmfdiXBemGxg0iTHREhs3gzPP0/jn3+Gvn0dMyNdZrq781YrL2dk8Mbx49iAsU2bMi4qigg5Uhd1kIS7cIrCxEJSBjuGDjgQHcHzmXF06ubF/70PrVoBZ8/C2LGwYgWEh3Ns7lyajh172QumX545w1OHDpFpNjOgUSOmN2tG8yoYU10IVyXhLmqU1WTl6LSjZM7OxNrAh1n12rH9dCgz5sKYMeXdG1evhpEjHdPdTZoEkyZRmJ5+yWBPLCxkypEjfH3uHG0DAtjesSM95K5SISTcRc3J35lPypAUSnQJB6Ib82JmHB26ebNvCSgFZGc7esGsWeMYS2DjxvKT7hereLE02MtLRmoU4g8k3EW1s563kj4xneMLjlPa0JeEgPbsOR3y29G6hx2WLnNcJC0pcVwwHTcOLnFT0enSUqZnZPBudjY+BgOTY2J4NiqKBnIDkhC/I78RolrlfpeLHqoxpZtIjIpkclZzet/mxf7XIT4e+OorR5j/8AP06OGYJSk+/qL3OW+zMS0jgzlZWZRYrQyPjGRKTAyN5WKpEJck4S6qRUl6CUcmH+H0qtOYQ/2YFtCRfWcaMnehY54MQ14uDBgFq1Y5xld/80148smLJtCw2Gx8mpPDc2fPcjwnhwfCwpjRrBnxAQFOqkwI1yDhLqqU3Wbn+MLjpD+Xjs0Om8KimZsTQ+++nux7A+Jb2BynYF58EXJzHWOuT5hQ3u/xNyVWK0tPnmR2VhYZJhPNPT35vmNHGYJXiD9Jwl1UmYK9BRwadYjCnwrJjglhzFGFj48vqy4M+/LTLrj+Kdi7F2680XG0/usg7A5FFgsLjh9n/rFj5JSVcUODBsxv0YIWp0/TVoJdiD9Nwl1cNVupjYxpGWS+komtgTeLglvxydFwRj1lICEBGvia4ZkXHCM2RkY6+q4PGPC7ro1l5adfJqSnc9Rspl9ICC9ER9MzKAiDwYAxJ8eJFQrheq4q3JVSjYBE4BbAAiwD7EASMFJrbVNKDQNGlD+foLVed1UtFrVKUVIRxoFGzu8/T1J0Y17IbEGLDl7s2gDXdbXD1q2OLjEHDji6Oc6cCfXr/7p9sdXK+9nZzM7KIstsppX0VReiSlS6U7BSyht4FygpX/UaMElr3RMwAPcopRoDo4HuwK3ATKWUdG9wA3arncw5mSR2TiT/cCkJfu0Yn9OKKbO82LMHriveCp06wU03QX4+rF3rGBumPNgLLRZezcyk2a5djE5LI8bPj7Xt2nGgSxcJdiGqwNUcuc8B3gFeLF/uDGwrf7we6AtYgR1aazNgVkqlAe2BPVexX+FkJUdKSHkshfzv89FhobyQE891fX1IehuahxXCsKdg+XKIiXEE+uOPQ/lQALllZaw8dYppGRmctVjoWz5XaS8JdCGqVKXCXSn1GJCjtf5GKXUh3A1aa3v540IgCGgA5FfY9ML6ixiNxso0BZPJVOltXZWzarbb7ZjWmCiYWUiZxcA8D8XWwkaMn3qaB/vn4rNuM5Zp0/DMzeXsiBGcGT4cu78/ZGSQa7OxoriY5cXFlNjtXO/tzVshIbT39obsbIzZ2f9x3/I51w1Sc9Wp7JH7EMCulLoZ6Ah8ADSq8HwgkAcUlD/+4/qLtG7dulINMRqNld7WVTmj5tJTpehhmoK1BRwObMjE4lb0fMCPzLch9GAqjJwK27Y5er+sX0/oddcRCuwrLGTB8eOsOnMGs93Og2FhjIuKomtgIIb/MGvSH8nnXDdIzVcmMTHxss9VKty11r0uPFZKbQWeAGYrpfporbcC/YDvgN3ADKWUH+ALtMZxsVW4CLvNTs5nOaT+4xCleRYWe8Wxyaspb31k4KF+BTB9OsybBxERsGABjBgBPj5kmkyMTUtjzZkzBHh4MDgigqeaNKFNvXrOLkmIOqEqu0KOAxYrpXwAI7Baa21VSi0AtuO4eDtRa22qwn2KapS3PY/U4akUpxST5V+fKZaOtLuzHgcWQcTPX0GLwZCT4wj0uXOhXj1OlZYyKy2Nt0+cwAC8HBvLqCZNaCgTZQhRo6463LXWfSos9r7E84uBxVe7H1FzKg70VVTfj9dpTWpwGPNWenDvzYUYnhnrGAPm2msdY8N07UqRxcK8jAxeyczEbLPxSOPGvBQbS4yfn7PLEaJOkpuYxO/kfZ9HypAUTIdNrPdrwlvnmzHiGS/WvASB+7bDXwbDkSOO4QOmTsXs7c2iY8dIOHqU02Vl3B8ayszmzWkpY78I4VQS7gKAsrNlHH7uMCeXnuScnx/T6YBfh2B+eNtOR+8keHSyY2Lq6GjYuhVrjx6sPHWKqUeOcNRspk/DhnzRrBk3Bl2yM5QQooZJuNdxdrudUytOceiZw5TlWvjMK4pPvWOZPs+T4fFb8XhkFBw8CPXqQUIC9qef5vPiYibt2YOxuJjO9euzSCluCQ6+ot4vQojqJeFehxWnFpP6RCp53+WR4tmA2bZ4ujxQn18mnaPJ6+Phyfcdk1EvXIj93ntZ7eHBK0YjPxcVofz9+bRNG+4PC5NQF6IWknCvg2xmG5mvZJIx4ygmuwcLiaewRwSfzYOOqR/DzaMdE1SPH499yhT+XVzM7KwsdhYUoPz9eV8pHg0Px0umtBOi1pJwr2Nyt+aSMiwVc1oJ33k0YllAHGNe8mFMy6/xHF0+I1LnztjXr2d9TAxTjEYSi4qI9vXlnfh4hkZE4ClH6kLUenLoVUeUnS3D+FgK+/+6n8x0O88b2pP+cBt2bzzOM1vuxPOeOyEzE+bP55v16/mLzcYdBw5wzmJhqVIcvv56RkRGSrAL4SLkyL0OyFmTQ/KwVCy5Fj4imvQbYlg5XtNy/QTovcQxEfX8+RwcNIhnMzPZcPAgzf38WKIUA8PD8ZbTL0K4HAl3N1aUVETaxAzyvjzDIeqzKLgD46fk88C3f8fw93Xg6wtDhlA4aRIvmUy8vm8fgV5ezI2LY2STJvhKqAvhsiTc3ZCtzEbW7CzSp2Zgthr4kGaEDAljW9MFBEx/FcxmePll7MOHs9bDg6cOHSLLbGZYRAQzmjUj9A/zmQohXI+Eu5vJ25ZH0rBULIeK2UYYW9q34P2B62n59jOOO0tvugnbwoV8FhzMlIwMUoqLiff3Z0enTnIDkhBuRP7vdhNl58rY/7CRfX32ceyQlRkB7ej49Bm2hfSj5fh7ISAAvv2Wn7/4gt7FxTyYnIwHsDg+nv1dukiwC+Fm5MjdDZz87AwHBqfiUVjKR0TT4G4P1jISv/mfQGQkttdfZ03//iw4eZLtiYk08PTkfaUY1Lix9H4Rwk1JuLsw83EzPz2Uim3HWY5Sj103xDO9y2IilySAzUZxQgLLBgwg4cQJsrUmzs+P2c2bMyQighAZglcItybh7qKyPjlD8qAUbCYbXzRozqDRyQz9eAC8eYiS/v15Y+JEZhUWci4jgx5BQSxo0YJ7w8LkSF2IOkLC3cWUniol5fFign9MIpP6lN5WwpteY/BO+ILsLl34YNMmZvr6kp+bS7+QEJ6PjqZXUJCM/yJEHSPh7iJsFhvGhOMcn5FBoMXGzpAAhnecQ+iGjykLCWHG0qXMiYsjz2qld/36TGvWjN4NGzq72UIIJ5FwdwEF+4rYeU8KfplFHDD4E9vtB15MfQXTbhPzFy1i8bXXkmwycUdQELPi4mgr85QKUedJuNdiNouN/5uQxdk5GRTbPclsms5o23S8dx5ly4MPMmzkSNJtNjp5ebG6bVvuDwtzdpOFELWEhHstlb+3kJ39U/HPKCTT8wR3Rs4jLGsv2++6i8mLF7ItIICWvr58Fx9Pn+BgZzdXCFHLSLjXMmV5ZeweeRTzqmOUYSYweBWD8lawpU1f5rz7Nd/4+9PQYOClmBiei4oiwNPT2U0WQtRClQp3pZQ3sASIBXyBBCAZWAbYgSRgpNbappQaBowALECC1nrd1TfbPWV9cBrjiFR8TBa0bw4DgyZwOMhAt/dW81NICI28vZkVFcUtRUV0io11dnOFELVYZYcfGAic1Vr3BPoBbwKvAZPK1xmAe5RSjYHRQHfgVmCmUsr36pvtXswnzWzsdpDDg5LJNR2nXuxSbg0Zw/NDH6TzokUcDg9ncXw8mTfeyPjoaPykW6MQ4r+o7GmZT4HVFZYtQGdgW/nyeqAvYAV2aK3NgFkplQa0B/ZUcr9uxW6zk/rPY2S9dASDtYyy+v+mp9+/mNp/MEv6fYjBYOAfTZowPTaWYLmjVAhxBSoV7lrrIgClVCCOkJ8EzNFa28tfUggEAQ2A/AqbXlh/EaPRWJmmYDKZKr2tM1lzrGQ8eZ76ycXksZ+2TT9g3kM30e/2jzF4eDAgIIAhAQGEWyycTEvjZIVtXbXmqyE11w1Sc9Wp9AVVpVQU8DmwUGu9Sin1aoWnA4E8oKD88R/XX6R169aVaofRaKz0ts5gt9v5eeYpTk47TEhpCtd4v0vy7ZE8MXQ6efXq8Y+mTRndpAktAgIu+x6uVnNVkJrrBqn5yiQmJl72ucpeUA0HNgKjtNbflq/+RSnVR2u9Fcd5+O+A3cAMpZQfjguvrXFcbK2TUjacZ/djabQ8ZaQFH5Le9TQjRo0iKTqavwYFMa9lSzrUr+/sZgoh3EBlj9wnAMHAZKXU5PJ1Y4AFSikfwAis1lpblVILgO04Lt5O1FqbrrbRruZcYhFbBh4hLCWTbnxEWdQOJox4nDXduxPn48Py5s15JDxcxn8RQlSZyp5zH4MjzP+o9yVeuxhYXJn9uDpLgYWtgzPwWJNBPF8RHLyGuQ/fxVv3vYevpycvx8TwXHS0zFUqhKhychNTNbDb7RjfzuHw2EPElG4h1Hc5/7r7LyQMfocCf38ej4hgemwsjX2lV6gQonpIuFexvOQSttxziKi0PXT2WMDmvo15adgrHAkN5bbgYGbHxdFOzqsLIaqZhHsVsZlt7Ho6E9M7B+nIUn7umcNjQ8eRHB1Np3r12BAXx60hIc5uphCijpBwrwLpn+WSNDSVZnmbyO20iYeG/y97W7VC+fryaVwc94WF4SEXS4UQNUjC/SoUZJSy7o40mif/gqXNRp6d0I6NXacRbTCwND6egeHheMnFUiGEE0i4V4KtzMaW0ScwvLubwFZfMXWWYuN1gwktLWV2TAyjoqPxk9EahRBOJOF+BWwWG0mvZHPin7uxRX/FG/+MZ8MNj3BNaSmzQkP5R6tW1PeSH6kQwvkkif4Eu91Oxqqz7H/qIIXN17Ps5XC2dB5IiNnMzMBARnXoIKEuhKhVJJH+i8Kk8+z8n0MUlG3i1RkN2dv6NhoXnWdWSAhPtmlDoIS6EKIWkmS6DEuhhe+HHsXjy2/5fEg2i+/rQVjReRZ7ePDI7f3krlIhRK0m4X4JvyzPI3PMFg53/5H5y7pwLKwFI/LymXnH7TSUu0qFEC5Awr2CvHQzax5MIrn1FlYuieNUyB20Lyjk41atuDEy0tnNE0KIP03CHSjKtfHlgAysx1bw2tMxHGzelZuPZvFx27b06t1bRmsUQricOh3uVqudlU+eJTvxW9Y9ms+ODr2JyCvga39/+g16xNnNE0KISquz4b7x3SKS5mxhx0MZrJnbniZnvHi9qIihd95OgPSAEUK4uDqXYvlZpcwdtYktvU+zY3EzvMtaM/VQGi/edy++11zj7OYJIUSVqDPhnp9dRsL07WzodJKksZFEnglk5oEUBt57J01vucXZzRNCiCrl9uFut9lZ8ey/eSMuj70PxRKfaWXW5mTGjOmPb1iYs5snhBDVwq3D/afZXzPv7AE+vvt6Gp2DV78+wLhnH8Lj0UbObpoQQlQrtwz3zZ8ks+rH9XzStw2lXp15dPMxZo26h8b3BTq7aUIIUSOqPdyVUh7AQqADYAaGaq3TqmNfu74yMvvH71lzs8Lrjg78becJJvS7jV4JN1fH7oQQotaqiQFS/g74aa1vBF4A5lb1DrZ9mkT/CQvp5XOMtb2b89gX+9hRHMmGKY/Sq6ucghFC1D01cVqmB7ABQGu9SynVpareuMxs4bWFW1h5ZwvsfVrwwOZURl7/V7rPl94vQoi6rSaO3BsA+RWWrUqpKvmj8uWiXbx3f1v+tiud7dZYVs0aSff72lTFWwshhEsz2O32at2BUuo1YJfW+pPy5WNa66YVX5OYmGgPCAio1PsfPZhDTNu61aXRZDLh5+fn7GbUKKm5bpCar0xxcTGdO3e+5OBXNXFaZgdwF/CJUuoG4MClXtS6detK7+BqtnVFRqNRaq4DpOa64WpqTkxMvOxzNRHunwO3KKV2AgZgcA3sUwgh6rRqD3ettQ14orr3I4QQ4jcyV5wQQrghCXchhHBDEu5CCOGGJNyFEMINSbgLIYQbqvabmP6MxMRE5zdCCCFc0OVuYqoV4S6EEKJqyWkZIYRwQxLuQgjhhlx2JqaanASkpimlvIElQCzgCyQAycAywA4kASO11jal1DBgBGABErTW65zR5qqilGoEJAK34KhpGW5cs1LqReBuwAfH93kbblxz+Xd7OY7vthUYhht/zkqp64FZWus+SqkW/Mk6lVL+wEqgEVAIDNJa51zJvl35yL3aJwFxooHAWa11T6Af8CbwGjCpfJ0BuEcp1RgYDXQHbgVmKqV8ndTmq1b+i/8uUFK+yq1rVkr1AbrhqKU3EIWb1wzcDnhprbsB04EZuGnNSqnxwHvAhSEfr6TOJ4ED5a/9AJh0pft35XD/3SQgQJVNAlILfApMrrBsATrjOKoDWA/cDFwH7NBam7XW+UAa0L4mG1rF5gDvACfKl9295ltxjJL6ObAWWIf715wKeJX/590AKMN9az4M3Fdh+Urq/DXfKrz2irhyuFfbJCDOprUu0loXKqUCgdU4/mobtNYXujYVAkFc/DO4sN7lKKUeA3K01t9UWO3WNQOhOA5K+uMYXO9DwMPNay7CcUomBVgMLMBNP2et9Wc4/nhdcCV1VlxfqdpdOdwLgMAKyx5aa4uzGlPVlFJRwHfACq31KsBW4elAII+LfwYX1ruiITiGht4KdMTxr2jFCXDdseazwDda61KttQZM/P6X2B1rHouj5ngc18uW47jecIE71nzBlfwOV1xfqdpdOdx34Dh/x3+aBMQVKaXCgY3A81rrJeWrfyk/RwuO8/Dbgd1AT6WUn1IqCGiN40KNy9Fa99Ja99Za9wH2AY8C6925ZuAH4DallEEpFQnUA75185pz+e2I9BzgjZt/tyu4kjp/zbcKr70irnwaw50nAZkABAOTlVIXzr2PARYopXwAI7Baa21VSi3A8cF7ABO11iantLh6jAMWu2vN5b0ieuH4BfcARgJHcOOagXnAEqXUdhxH7BOAvbh3zRf86e+zUuptYLlS6gegFHj4Sncmd6gKIYQbcuXTMkIIIS5Dwl0IIdyQhLsQQrghCXchhHBDEu5CCOGGJNyFEMINSbgLIYQbknAXQgg39P+3D/aBaKFEyQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#The 3 plots above all on same set of axes\n",
    "plt.plot(reward_arr1, label = \"Original\", color = \"b\")\n",
    "plt.plot(reward_arr2, label = \"3/2 Replaced by 1\", color = 'm')\n",
    "plt.plot(reward_arr3, label = \"3/2 Replaced by 3\", color = 'r')\n",
    "plt.plot(reward_arr4, label = \"ln(t) Replaced by t\", color = 'c')\n",
    "plt.title(\"Cumulative Rewards\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Record 2 things for every 20 runs in time sequence\n",
    "#i.e. 1. selection distribution of each arm; 2. sum of returns of these 20 runs\n",
    "#so that you have 50 sets of records\n",
    "#implementing UCB, 1000 runs\n",
    "#CMAB for 2 arms being selected\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for i in range(1):\n",
    "    for t in range(20):\n",
    "        UCB_Values = np.zeros(k) #array holding the ucb values. we pick the max \n",
    "        temp_UCB_Values = np.zeros(k)\n",
    "        arm_selected = 0\n",
    "        for a in range(k):\n",
    "            if (N[a] > 0):\n",
    "                #calculate UCB below\n",
    "                mu = rewardSum[a]/N[a]\n",
    "                ucb_value = mu + math.sqrt((3/2)*math.log(t)/N[a]) \n",
    "                UCB_Values[a] = ucb_value\n",
    "            elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "                UCB_Values[a] = 1e500 #infinity\n",
    "            temp_UCB_Values[a] = UCB_Values[a]\n",
    "\n",
    "        arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "        temp_UCB_Values[arm1_selected] = 0\n",
    "        arm2_selected = np.argmax(temp_UCB_Values)\n",
    "\n",
    "        #update Values as of round t\n",
    "        N[arm1_selected] += 1\n",
    "        N[arm2_selected] += 1\n",
    "        reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "        rewardSum[arm1_selected] += df.values[t, arm1_selected+1]\n",
    "        rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "        hist_UCB_rewards.append(reward)\n",
    "        UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "        UCB_total_reward = sum(hist_UCB_rewards)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD0CAYAAABdAQdaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAf3UlEQVR4nO3deZyN5eP/8dfMWCZCWmx9VFpcSSEjSZTsPkK2LI1EZZ2EfMjWyqMkIilZJoqxZ9+G5KNSaKxpXEJ99BWSrQZjzMz9+2NGv2nsM3POfZb3868z933OmbfL7e0617nPuUMcx0FERHxbqNsBRETk8lTWIiJ+QGUtIuIHVNYiIn5AZS0i4gdU1iIifiCXp544Li5O5wSKiGRBRERESOZtHivr9F+YpcfFx8dTpkyZHE6Tc5Qve5Qve5Qv+3w5Y1xc3AW3axlERMQPqKxFRPyAylpExA+orEVE/IDKWkTED6isRUT8gMpaRMQPqKxFRHKA4ziM3TCWku+V5Jfjv+T486usRUSy6c8zf9J6bmuilkVRvmh5bi5wc47/Do9+glFEJNBtPbiVlrNbsvfYXobVHkafqn0IDcn5ebDKWkQkCxzHIXpzNFHLoigcXpjV7VfzyK2PeOz3qaxFRK7SyaSTdF/anSlbp1CrVC1imsdQJH8Rj/5OlbWIyFWIPxxPy9kt+fHwj7z66KsMfmQwYaFhHv+9KmsRkSsUsz2GTos6kS93PlZErqDOHXW89rtV1iIil5GYnEjP5T35OO5jqt1SjRnNZ3BzwZw/4+NSVNYiIpew5+geWs5uyeaDm+n3cD+G1BxCrlDvV6fKWkTkIj6P/5wOCzoQFhLGojaLeLz0465l0YdiREQySUpJotfyXjSf1Rxzg2FT502uFjVoZi0i8g/7Tuyj1ZxWfPd/3/FC5Rd4t+675AnL43YslbWICKR9yCVmeww9lvfgbMpZZrWYRcuyLd2O9bdLlrUxJjcQDdwG5AWGAD8CkwEH+AHobq1N9WhKEREP2ntsL12XdCV2TywP3vwgnzX9jLtuuMvtWP9wuTXrSOCItbY60AD4ABgJDErfFgI08WxEERHPOJtylne+eYd7P7yXb3/9ljENxvBNx298rqjh8ssgs4E5GX5OBiKA/6b/vAyoC8zL+WgiIp6zcf9Gnl/0PFsPbeWJu59gTIMx/Kvgv9yOdVEhjuNc9k7GmALAQmAC8K61tkT69ppAR2ttZObHxMXFOfny5ctSqMTERMLDw7P0WG9QvuxRvuxRvuw5efYkI7eMZMbPM7jpmpsYdP8gav+rttux/nbq1CkiIiJCMm+/7BuMxpiSpM2cP7TWxhhj3smwuwBw/GKPLVOmTFayEh8fn+XHeoPyZY/yZY/yZd0iu4huX3Rj/5/76fZAN4bWHEqh8EJux/qHuLi4C26/5Jq1MaYoEAv0s9ZGp2/ebIypkX67AfBVDmUUEfGI3/76jRazWtB4RmOuC7+OaTWn8cG/P/C5or6Uy82sBwCFgcHGmMHp214E3jfG5AHi+eeatoiIz0h1UhkfN55+q/pxJvkMQ2sOpU/VPuzZtcftaFftkmVtrX2RtHLO7FHPxBERyRk7ft9Bp8WdWPfrOmqWqsm4huN88iyPK6UPxYhIQElMTmTo2qEM+2YYBfIWYHKTyTxd/mlCQs57z86vqKxFxO+lOqlsObiFpT8tZcrWKew+upvIcpGMrDuSm/Lf5Ha8HKGyFhG/dOz0MVbuXcnSn5ayfPdyDp08BMADJR4gNjLWqxcG8AaVtYj4Bcdx/p49L9u9jG//71tSnVQKhxem3p31aHBnA+rdUY+i1xZ1O6pHqKxFxGcdTzzOyj0rWbo7bfZ8MOEgABHFIxhQbQAN7mpA5Zsru3IxAG8L/D+hiPiVbYe2sWTXEpbuXsq3v35LipNC4fDC1L2jLg3ubED9O+sH7Oz5UlTWIuITfvvrN3os68Hc+LkAVCxekZervUyDOxvw4L8eDIrZ86UE959eRFyX6qQy7vtx9P+iP0kpSQx5bAjPVnyWYtcWczuaT1FZi4hrth/aTqfFnfju/76j9u21GddwHHdcf4fbsXySylpEvO702dO8ufZNhq8bznXh1/FZ08946r6n/P6DK56kshYRr1q1dxVdFndhz7E9PFPhGYbXGc6N+W50O5bPU1mLiFccPnmYl2Jf4rNtn3HX9XfxxdNfULNUTbdj+Q2VtYh4lOM4fLr1U16KfYk/z/zJoOqDGPjIQMJz+e4FCnyRylpEPOanIz/RZUkXVv+8mqolqzL+8fGULVLW7Vh+SWUtIjkuKSWJd755hyFrhxCeK5xxDcfxfMTzhIZc7hrdcjEqaxHJUZv+2ESLL1vw4+EfebLsk4yqN4riBYq7HcvvqaxFJMeMWT+GHqt7cEuhW1jcZjENSzd0O1LAUFmLSI4YsW4EfVb2odbNtZj/9HyuzXOt25ECihaQRCTb3vrqLfqs7EPLe1oy8qGRKmoPUFmLSJY5jsPra15nwOoBtL2vLTHNY8gdmtvtWAFJyyAikiWO4zD4y8EM/Woo7cu3Z1LjSYSFhrkdK2CprEXkqjmOQ79V/Ri+bjjP3f8cHzf6WKfleZjKWkSuiuM49FrRi9HrR9OtUjfG/HuMitoLVNYicsVSnVSilkbx0fcf8eKDL/Jevff0TXleorIWkSuS6qTSeVFnJm6eyH+q/odhtYepqL1Ir11E5LJSUlPouKAjEzdPZFD1QSpqF2hmLSKXlJyaTPv57YnZHsMbNd5g8KOD3Y4UlFTWInJRZ1PO8tTnTzH7x9m8VestXq72stuRgpbKWkQuKCkliVZzWjF/53xG1B1B74d6ux0pqKmsReQ8icmJtJjVgiU/LWFMgzFEVY5yO1LQU1mLyD+cPnuapjObsmLPCsY1HEfnSp3djiTobBARyeDXE7/SaHojYvfEMqnxJBW1D9HMWiTInUk+wwK7gOjN0cTuiSU0JJQpT0yhXfl2bkeTDFTWIkFqy8EtRG+OZtr2aRw9fZSSBUsy+JHBPFPhGUoVLuV2PMlEZS0SRI6ePkrM9hiiN0ez+eBm8oblpWmZpnSs0JGapWrqW/N8mMpaJMClpKbwxc9fEL05mnk755GUkkTF4hX5oMEHtLmvDddfc73bEeUKqKxFAtTPx35m8pbJTN46mX0n9lE4vDCdIzrT8f6OVChWwe14cpVU1iIBJDk1mZk/zCR6SzSrf15NCCHUvaMuw+sMp7FpTHiucLcjShaprEUCxNaDW+m4sCObDmyi1HWleKPGG7Sv0J5bCt3idjTJASprET93JvkMQ78ayltfv8X111zPzBYzaXFPC10QIMCorEX82Ib9G+i4oCM7Du8gslwko+qN4oZ8N7gdSzzgisraGPMgMMxaW8MYUxFYBPyUvvsja+1MTwUUkfOdPnuaV9e8yohvR1D82uIsbrOYhqUbuh1LPOiyZW2M6Qu0A06mb6oIjLTWjvBkMBG5sO8Pf0+TVU346ehPPF/xeYbXGU6h8EJuxxIPu5KZ9R6gGfBZ+s8RgDHGNCFtdt3TWvuXh/KJSLqEpAT6r+rPBxs/oNR1pVjVbhW1bq/ldizxkhDHcS57J2PMbcAMa20VY0wHYJu1Ns4YMxAobK3tk/kxcXFxTr58+bIUKjExkfBw3z3FSPmyR/mu3rqD63jl+1c4cOoArUu1pneF3uTPnd/tWBfki+OXmS9nPHXqFBEREeddMy0rbzDOs9YeP3cbGHOxO5YpUyYLTw/x8fFZfqw3KF/2KN+VO554nD6xfZi0eRKlbyjNV62+4vqT1/tMvgvxpfG7GF/OGBcXd8HtWTm3Z4UxpnL67VrAhZ9ZRLJlkV1E2Q/L8smWT+j3cD+2dN7Cw7c87HYscUlWZtZdgQ+MMUnAQaBTzkYSCW5/nPqDF5e/SMz2GO4rch8LWi+gUolKbscSl11RWVtrfwGqpN/eBFT1YCaRoLTryC6mbZvGR99/xPHE47z26Gv0r96fPGF53I4mPkAfihFx0aGEQ8zcMZOp26ay8beNhBBC7dtrM6LuCO4rep/b8cSHqKxFvCwhKYH5O+czbfs0Vu5ZSYqTQoViFXi3zru0vrc1Nxe82e2I4oNU1iJecDblLCv3rmTa9mnM3zmfU2dPcWuhW+n3cD+eKvcU99x0j9sRxceprEU8xHEcNuzfwNRtU5m5YyaHTx2mcHhh2pVrR2S5SKqWrKovW5IrprIWyWHn3iictn0ae47tIW9YXhqbxkSWi6T+nfX1hqFkicpaJIckpSTRJ7YPYzaMIYQQapaqycDqA2lWppm+u0OyTWUtkgN+++s3Ws5uybpf1/FC5Rfo93A/vVEoOUplLZJNa/+3lidnP0lCUgIzms+g1b2t3I4kAUjvbohkkeM4vPfte9ScUpNC4YVY/9x6FbV4jGbWIlmQkJTAcwufY+aOmTxx9xNMbjJZ69LiUSprkau068gums1sRvwf8bxd6236PtyXkJDzvtFSJEeprEWuwvyd83l63tPkzZWXFZErqH17bbcjSZDQmrXIFUhJTaH/qv40ndmUu2+8m7hOcSpq8SrNrEUu4/DJw7T9vC2r9q6iU8VOjG4wmvBcvnmVEQlcKmuRS9i4fyPNZzXn95O/M6nxJDre39HtSBKktAwichET4iZQ7ZNqhIaE8k3Hb1TU4irNrEUyOZNyhucWPsekzZOoc3sdYprHcGO+G92OJUFOZS2Swf+O/4/I1ZHsOLaDgdUH8nqN1wkLDXM7lojKWuSc2D2xtJnbhqTkJBa0XkBj09jtSCJ/05q1BL1UJ5Wha4dSf2p9ShQowazas1TU4nM0s5agdjzxOE/Pe5pFuxbR9r62jH98PPv27HM7lsh5VNYStLYf2k6zWc345fgvvF//faIqR+lj4+KzVNYSlKZtm8bzi57nuvDrWNN+DQ/f8rDbkUQuSWvWElSSUpLosawHkfMiqVSiEps6b1JRi1/QzFqCRsarufSq0othtYeROyy327FErojKWoKCruYi/k7LIBLQdDUXCRSaWUvASkhK4NmFzzJrxyxdzUX8nspaApL9w9J0ZlPsEauruUhAUFlLwPk8/nOemf8MeXPlJTYyllq313I7kki2ac1aAsZfZ/6i+5LuNJ/V/O+ruaioJVBoZi0BYcXuFXRa3IlfT/xKzwd78nbtt8mbK6/bsURyjMpa/Nqx08d4KfYlPtnyCXffeDdfd/yaqiWruh1LJMeprMVvLdi5gK5LuvL7yd/pX60/rzz6iq6NKAFLZS1+5/DJw/RY3oMZP8ygXNFyLGqziIgSEW7HEvEolbX4DcdxmLVjFlHLojiReII3arxBv2r9yBOWx+1oIh6nsha/cOCvA3Rb2o35O+fzQIkHiG4Szb1F7nU7lojXqKzFpzmOw5StU+i1oheJyYm8U/sdej3Ui1yhOnQluOiIF5+178Q+Oi3qxIo9K6h2SzUmNZ5E6RtKux1LxBUqa/E5qU4q4+PG85+V/8FxHMY0GEO3B7oRGqLPcEnwuqKyNsY8CAyz1tYwxtwJTAYc4Aegu7U21XMRJZjsOrKLzos7s+aXNdQqVYsJjSZQqnApt2OJuO6yUxVjTF9gInDuBNaRwCBrbXUgBGjiuXgSLBKSEnh51cvc++G9bDqwiYmNJrKy3UoVtUi6K5lZ7wGaAZ+l/xwB/Df99jKgLjAv56NJMHAch5k7ZtIntg/7/9rPMxWe4e1ab1P02qJuRxPxKZcta2vtXGPMbRk2hVhrnfTbfwEX/YLg+Pj4LIVKTEzM8mO9Qfmy51y+Xcd3MXTzUDYe3sg9he9heM3hVLixAkd/PcpRjrqez1cpX/b5Q8bMsvIGY8b16QLA8YvdsUyZMll4+rSSz+pjvUH5smf91vVM/mUyYzeOpVB4IcY1HMdzFZ8jLDTM7WiA74+f8mWfL2eMi4u74PaslPVmY0wNa+0aoAHwZTZySRBJdVKZsmUKfVb04diZY3Sp1IU3H3uTG/Ld4HY0EZ+XlbJ+CZhgjMkDxANzcjaSBKLvf/ueqKVRrN+/ngo3VGBl+5VULF7R7VgifuOKytpa+wtQJf32LuBRD2aSAPLHqT8Y8MUAJm6aSJH8RZjyxBQickdQtnhZt6OJ+BV9KEY8IiU1hY/jPmbQ6kH8eeZPelXpxSuPvkKh8EJ+98aOiC9QWUuO+3rf17yw7AW2HNxCzVI1eb/++5Qtopm0SHaorCVHOI7Dqr2rGLV+FEt/WkrJgiWZ3XI2zcs011XFRXKAylqy5fTZ00zdNpXR60ez4/AOiuYvypuPvUmvKr3Inye/2/FEAobKWrJk/5/7+XDjh3wc9zFHTh+hQrEKTHliCq3KttKFakU8QGUtV2Xj/o2MWj+KWTtmkZKaQpO7m9CrSi+q31Jdyx0iHqSylstKTk1m/s75vPfde6z7dR0F8hQg6oEoXnjwBW4vfLvb8USCgspaLup44nEmbprImA1j2HdiH7cXvp1R9UbR4f4OFMxb0O14IkFFZS3n2XVkF++vf5/JWyZz8uxJatxWg/frv8/jpR/3me/vEAk2Kmv526GEQ/Rd1ZdPt35KnrA8tLm3DT2r9KRCsQpuRxMJeiprITk1mY82fsSgLwdx+uxp+lbtS6+HelHs2mJuRxORdCrrIPf1vq/pvrQ72w5to+4ddRnTYIwuSivig1TWQSrjkscthW5h7pNzaXp3U51+J+KjVNZBJvOSx4BqAxhQfYA+bSji41TWQURLHiL+S2UdBLTkIeL/VNYBTEseIoFDZR2gtOQhElhU1gHmYMJB+q/vz4L/LdCSh0gAUVkHiDPJZ3h//fu8ufZNTidryUMk0Kis/ZzjOCzetZjesb3ZfXQ3jUo3ousdXWlQuYHb0UQkB4W6HUCy7sfDP1J/Wn0az2hM7tDcLH9qOQvbLOS2Are5HU1Ecphm1n7o2OljvLbmNcZuHEuBvAUYXX80XSt1JXdYbrejiYiHqKz9SHJqMhPiJjD4y8EcSzxG54jOvPHYG9yY70a3o4mIh6ms/cSXP3/Ji8tfZPvv26lxWw1G1x9NuaLl3I4lIl6isvZxPx/7mT4r+/B5/Ofcdt1tzGk5h2ZlmulUPJEgo7L2UQlJCbz11VuM+HYEYaFhDHlsCL0f6s01ua9xO5qIuEBl7WNSnVSmbZtGv1X9OJBwgMhykbxd621uLniz29FExEUqax+SkppC+/ntmbZ9Gg+UeIC5T87loZIPuR1LRHyAytpHpDqpPLvwWaZtn8YbNd5g4CMDCQ3RafAikkZl7QNSnVS6LO7ClK1TeL3G6wx+dLDbkUTEx2jq5jLHceixrAcTNk1gQLUBDH5ERS0i51NZu8hxHHqv6M3YjWPp81AfhtQcolPyROSCVNYucRyH/l/0Z9T6UfSo3IN36ryjohaRi1JZu+S1Na8x7JthdInowqj6o1TUInJJKmsXDFk7hDfWvkHHCh0Z23CsilpELktl7WXDvxnO4C8H065cO8Y3Gq/T80TkiqgpvGj0d6Ppu6ovrcq2IrpJNGGhYW5HEhE/obL2kg83fkjPFT1pVqYZnzX9jFyhOsVdRK6cytoLJm6aSPel3WlUuhHTm0/XRQJE5KpleXpnjNkMnEj/8WdrbYeciRRYPt36KZ0WdaL+nfWZ3XI2ecLyuB1JRPxQlsraGBMOYK2tkaNpAsz07dPpsKADtW6vxedPfk7eXHndjiQifiqrM+vyQD5jTGz6cwyw1n6Xc7H839wf59JuXjuq31KdBa0X6HuoRSRbQhzHueoHGWPuA6oAE4G7gGWAsdYmn7tPXFycky9fviyFSkxMJDw8PEuP9YbL5Vu9fzU91/Wk3A3l+Lj6x+TPnd+L6fx//NymfNnj6/nAtzOeOnWKiIiI8z58kdWZ9S5gt7XWAXYZY44AxYFfM96pTJkyWXry+Pj4LD/WGy6Vb/7O+fT6thcVS1RkZbuVFMxb0Mvp/Hv8fIHyZY+v5wPfzhgXF3fB7Vk9G6QjMALAGFMCKAgcyOJzBQTHcRiydghNZzbl/uL3s/yp5a4UtYgEpqzOrCcBk40xXwMO0DHjEkiwSUhKoMOCDsz5cQ6R5SIZ//h4rVGLSI7KUllba5OAtjmcxS/9cvwXmsxowg+//8C7dd6l90O99V0fIpLj9DG6bFjzyxpazGpBipPC0rZLqXdnPbcjiUiA0icYs8BxHMZuGEvtT2tTJH8RNjy3QUUtIh6lmfVVSkpJ4rW415i9dzaNSjdiarOpeiNRRDxOM+urcCjhEDWn1GT23tkMrD6Q+a3nq6hFxCs0s75Ccb/F8cTMJzhy6ggjHhpB75q93Y4kIkFEZX0FYrbH8OzCZymSvwjrnl1H3mP6jg8R8S4tg1xCSmoK/Vb246nPn6LyzZX5/vnvqVCsgtuxRCQIaWZ9EccTj9N2bluW7V5G10pdGVV/lL7eVERco7K+gJ1/7KTJjCbsPbaXcQ3H0blSZ7cjiUiQU1lnstAupN28duQNy8vqp1dT/dbqbkcSEdGa9TmJyYn0WNaDJjOacOf1d/J9p+9V1CLiMzSzJm3Zo/Wc1mw9tJWeD/bk7dpv66ouIuJTgrqsHcdh8pbJRC2LIl/ufCxqs4jHSz/udiwRkfMEbVmfSDxB1yVdmf7DdB677TGmNptKiQIl3I4lInJBQVnWG/ZvoPWc1uw7sY8hjw3h5WovExYa5nYsEZGLCqqyTnVSeXfduwxcPZASBUqwtsNaqpas6nYsEZHLCpqyPphwkPbz2xO7J5bmZZozodEECl9T2O1YIiJXJCjKOnZPLO3mtePPM38yruE4OkV00tVcRMSvBHRZJ6UkMWj1IIavG849N93DF09/wb1F7nU7lojIVQvYst57bC+t57Rm428b6RzRmZH1RpIvdz63Y4mIZElAlvX07dPpvLgzoSGhzG45mxb3tHA7kohItgRUWR89fZQXl7/I1G1TqVqyKjHNYrj1ulvdjiUikm0BU9bzd86n65KuHD55mMGPDOaVR18hV2jA/PFEJMj5fZv9ceoPeizrwfQfplO+aHmWtl3K/cXvdzuWiEiO8uuynvPjHLov7c7R00d5vcbrvFztZV0gQEQCkl+W9e8nfydqaRSzf5zN/cXuJzYylvLFyrsdS0TEY/yqrB3HYdaOWUQti+JE4gmGPDaEvg/3JXdYbrejiYh4lN+U9cGEg3Rb0o15O+dRqUQlPmnyiT7gIiJBw+fL2nEcYrbH0GN5D04mneTtWm/zUtWXdKaHiAQVn268A38doMuSLiy0C6nyrypEN46mzE1l3I4lIuJ1PlnWjuMwZcsUeq7oSWJyIu/WeZeeVXrqO6dFJGj5XFn/fvJ3un7dlbUH1vJwyYeJbhJN6RtKux1LRMRVPlfW07dPZ8PvGxhVbxRRlaM0mxYRwQfLutsD3aievzoV76vodhQREZ8R6naAzHKH5eaaXNe4HUNExKf4XFmLiMj5VNYiIn5AZS0i4gdU1iIifkBlLSLiB1TWIiJ+QGUtIuIHQhzH8cgTx8XFeeaJRUQCXEREREjmbR4raxERyTlaBhER8QMqaxERP+DqFzkZY0KBD4HywBngOWvt7gz7GwGvAMlAtLV2gpfz5QaigduAvMAQa+3CDPt7A88Ch9M3dbbWWi9n3AycSP/xZ2tthwz73B6/Z4Bn0n8MByoAxay1x9P3uzZ+xpgHgWHW2hrGmDuByYAD/AB0t9amZrjvJY9TL+SrAIwBUtJ//9PW2kOZ7n/R48AL+SoCi4Cf0nd/ZK2dmeG+bo/fDKBY+q7bgO+sta0z3d+r45cVbn/r3hNAuLX2IWNMFWAE0AT+Lsr3gAeAk8A3xphF1tqDXswXCRyx1rYzxtwAbAYWZthfkbR/OHFezPQ3Y0w4gLW2xgX2uT5+1trJpJUgxpixpP2HcTzDXVwZP2NMX6AdaeMCMBIYZK1dY4wZR9oxOC/DQy56nHop32jgBWvtFmNMZ6Af0DvD/S96HHgpX0VgpLV2xEUe4ur4nStmY0xh4EugV6b7e3X8ssrtZZBqwHIAa+13QKUM+8oAu621x6y1ScDXQHUv55sNDM7wc3Km/RFAf2PM18aY/t6L9bfyQD5jTKwxZnX6P4RzfGH8ADDGVALKWmvHZ9rl1vjtAZplyvHf9NvLgNqZ7n+p49Qb+Vpba7ek384FJGa6/6WOA2/kiwAaGmPWGmMmGWMKZLq/2+N3zuvAGGvtgUzbvT1+WeJ2WRfk/7/0AEgxxuS6yL6/gELeCgZgrU2w1v6VfvDNAQZlussMoAtQE6hmjHncm/mAU8C7QL30HNN8afwyGEDaP5TMXBk/a+1c4GyGTSHW2nOnRV1onC51nHo837lyMcZUBaJIe8WU0aWOA4/nAzYA/7HWPgLsBV7N9BBXxw/AGFMEqEX6K71MvDp+WeV2Wf8JZPxfONRam3yRfQWAjC+hvcIYU5K0l06fWWtjMmwPAUZZa/9In7kuAe73crxdwFRrrWOt3QUcAYqn7/OV8bsOuNta+2Wm7b4wfuekZrh9oXG61HHqFcaYVsA4oKG19nCm3Zc6DrxhXoalrHmc//fo+vgBLYAYa23KBfa5PX5XxO2y/gb4N0D6S4/tGfbFA3cZY643xuQBHgG+9WY4Y0xRIBboZ62NzrS7IPCDMeba9OKpCXh77bojaet/GGNKpGc69xLP9fFL9wiw6gLbfWH8ztlsjKmRfrsB8FWm/Zc6Tj3OGBNJ2oy6hrV27wXucqnjwBtWGGMqp9+uxfl/j66OX7rapC1xXYjb43dF3J7qzwPqGGPWASFAB2NMW+Baa+349LMFVpD2n0q0tXa/l/MNAAoDg40x59auJwD50/MNIG3WfQb4wlq71Mv5JgGTjTFfk3YmQ0fgSWOMr4wfgCHtpXHaD//8+3V7/M55CZiQ/p9aPGlLXhhjPiVt6eu849RbwYwxYcD7wD7gc2MMwH+tta9myHfeceDlmWtX4ANjTBJwEOiUnt318cvgH8ch/COf2+N3RfQJRhERP+D2MoiIiFwBlbWIiB9QWYuI+AGVtYiIH1BZi4j4AZW1iIgfUFmLiPgBlbWIiB/4f6kpn9kjU1aPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(49):\n",
    "    for t in range(1,21):\n",
    "        for a in range(k):\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt((3/2)*math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "            temp_UCB_Values[a] = UCB_Values[a]            \n",
    "\n",
    "        arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "        temp_UCB_Values[arm1_selected] = 0\n",
    "        arm2_selected = np.argmax(temp_UCB_Values)\n",
    "\n",
    "        #update Values as of round t\n",
    "        N[arm1_selected] += 1\n",
    "        N[arm2_selected] += 1\n",
    "        reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "        rewardSum[arm1_selected] += df.values[t, arm1_selected+1]\n",
    "        rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "        hist_UCB_rewards.append(reward)\n",
    "        UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "        UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD0CAYAAABgk2Y8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de4BN5f7H8bfbjGhGiKic1JHnjNylcldHRIrukQzKLY5SIceoiJBbDbk0CF3OOZ0UIZdkTGMSNeTW9tQolJTLZIyYYWb27489+s1xycyePbNm7/15/bX3s9ayvl/Dx+NZe69VzO12IyIigaW40wWIiIjvKdxFRAKQwl1EJAAp3EVEApDCXUQkACncRUQCUEmnCwBITEzU5zFFRLzQqFGjYucbLxLhDtCoUSOvjnO5XERERPi4mqJNPQcH9Rwc8tNzYmLiBbdpWUZEJADlauZujLkZmGCtbZ1jrCvwD2ttk+z3vYG+QAYwxlq7zBhzCfA2UBlIBSKttYd824KIiJztojN3Y8xQYA5QOsdYfeAxoFj2+yrAIKAZ0A4YZ4wJBfoD2621LYCFQJSvGxARkXPlZllmN3DvmTfGmIrAeOCpHPvcBCRYa9OttSlAElAXaA6szN5nBdDGF0WLiMifu+iyjLV2kTGmOoAxpgQwFxgMnMyxWziQkuN9KlDurPEzY+flcrnyUvcf0tLSvD7WX6nn4KCeg0NB9ZzXT8s0Aq4HZuJZpqlljHkVWAuE5dgvDDgKHMsxfmbsvLy9Wqyr68FBPQcH9Zw3f/ZpmTyFu7V2E3ADQPZs/t/W2qey19zHGmNKA6FABLADSAA6AJuA9kC8F/WLiEge+eSjkNbaX4BoPOG9FhhhrU3DM8O/wRizHugDjPLF+URE/N0Hrg+4ftr1fLzv4wL59XM1c7fW7gFu+bMxa20MEHPWPieAB/JbpIhIoPjl+C8M/Hggi1yLaFClAfUq1iuQ8+hLTCIihcDtdrPg6wXUer0Wy75dxri/j2Pj4xu5quxVBXK+InP7ARGRQLX36F76LuvLqt2raP6X5sy5aw7mclOg51S4i4gUkCx3FjO+nMFza54DYHr76fRv3J/ixQp+0UThLiJSAJKSk+i5pCfr962n3V/bMbvjbK657JpCO7/CXUTEh9xuN28kvsHTq58mpEQI8zvNp3u97hQrdt478xYYhbuIiI/sObqHfsv6sWr3Ktpc14Y3O73J1eFXO1KLwl1EJJ8yszJ5beNrjIwdSfFixZnWfhpPNH6iUNbWL0ThLiKSD98d+Y6eS3qS8GMCd9W8i9c7vE61ctWcLkvhLiLijSx3FtEboxn+6XBKlyzN2/e8Tdc6XQt9bf1CFO4iInn0c+rPPPbRY6xMWknHmh2Z3XE2V4Zd6XRZ/0PhLiKSS263m4VbF/LkyidJz0xnRocZ9LuxX5GZreekcBcRyYXkk8n0X96f93a+R6trWjHn7jnUqFDD6bIuSOEuInIRK5NW8thHj3Hw94O8fNvLDG02lBLFSzhd1p9SuIuIXEBqeirPrH6GmM0x1KpUi2VdltGgagOny8oVhbuIyHms27OOnkt6si9lH0ObDmXUraMoXbK002XlmsJdRCSHtIw0Rnw6gilfTOH6Ctezvud6mlRr4nRZeaZwFxHJtmn/Jnot6cXOQzt54sYnmNh2ImVKlXG6LK8o3EUk6GVmZTIhYQLPxz5P1bCqfNz1Y9pf397psvJF4S4iQW3nwZ30XtqbDT9t4OHaDzPzzplcVvoyp8vKN4W7iASlzKxMJn4+kedjnycsNIy37nmLR+o8UiS/kOQNhbuIBJ29R/fSfXF3Ptv7GfdF3MeMO2dQuWxlp8vyqVyFuzHmZmCCtba1MaY+MA3IBNKB7tbaX40xvYG+QAYwxlq7zBhzCfA2UBlIBSKttYcKohERkYtxu93M/3o+T658EoA3O71JZL3IgJmt53TRmw0bY4YCc4AzH/B8DfiHtbY18AEwzBhTBRgENAPaAeOMMaFAf2C7tbYFsBCI8nkHIiK5sPPgTu589056fdSLBlUbsLXfVnrU7xGQwQ65CHdgN3BvjvcPW2u/zn5dEkgDbgISrLXp1toUIAmoCzQHVmbvuwJo45OqRURy6eTpkwxZPYS6s+qS8GMCr7Z7ldjIWK4tf63TpRWoiy7LWGsXGWOq53h/AMAY0xQYCLTEM1tPyXFYKlAOCM8xfmbsvFwuVx5L90hLS/P6WH+lnoODes6/nck7GbZpGN8f+54HrnuAwXUGc1noZdhd1mfnyK+C+jl7dUHVGPMQMAK401p7yBhzDAjLsUsYcBTIOX5m7LwiIiK8KQWXy+X1sf5KPQcH9ey9U5mneCnuJcatH8cVl17B6m6ruf2vt/ugQt/LT8+JiYkX3JbncDfGdMNz4bS1tTY5e3gTMNYYUxoIBSKAHUAC0CF7e3sgPq/nExHJi+2/bqfHkh5sPrCZyHqRTG03lfKXlHe6rEKXp3A3xpQAooF9wAfGGIA4a+0LxphoPOFdHBhhrU0zxswEFhhj1gOngK4+rV5EJNtvJ39jyCdDmLdlHhUuqcCiBxdxb8S9Fz8wQOUq3K21e4Bbst9WuMA+MUDMWWMngAfyUZ+IyEWt+G7FH/dbf7rJ0wxvPpyKZSo6XZaj9CUmEfFbqempDPlkCLMTZ1O7cm2Wd13uN/dbL2gKdxHxS2fut7736F6ebfIsL932kl/db72gKdxFxK+cOH2C59Y8x7RN06hRoQbxPeNp9pdmTpdV5CjcRcRvfP7j50QujiQpOYl/3PQPxv19HGVDyjpdVpGkcBeRIi8tI42Ra0cyecNkrrnsGtZ2X8ut197qdFlFmsJdRIq0Tfs3Ebk4kl2Hd9GnYR8mtZ1EWGjYxQ8Mcgp3ESmS0jPSGR03mgkJE6hyaRVWPrKSdjXaOV2W31C4i0iRs/nAZnos7sH2g9vpUb8HU9tNDYinIxUmhbuIFBmnMk8xcu1Ixq0fR+Wylfno4Y+4y9zldFl+SeEuIkVC4s+JdFnThe9SvqN7ve682u7VoLwnjK8o3EXEUekZ6bz02UuMXz+eCqEVWNplKR1rdnS6LL+ncBcRx2z9ZSvdPuzGjoM7iKwXSb9r+3FLzVsufqBcVG6exCQi4lOZWZlMWD+BxjGNOXziMEu7LGV+5/mUC7ng83wkjzRzF5FCZQ9bei7pyYafNnBfxH3M6jiLy8tc7nRZAUfhLiKFIjMrk6lfTGVk7EguKXkJb93zFo/UeSRgH1DtNIW7iBS4XYd30XNJT7746QvuNncz685ZVA2r6nRZAU3hLiIF5sxsPWptFGVDyvLOve/QpXYXzdYLgcJdRApEztl6J9OJWR1nUeXSKk6XFTQU7iLiU5lZmUzZMIWRsSMpG1KWd+99l4drP6zZeiFTuIuIz+xO3k2PJT1Yv289nf/WmZl3ztRs3SEKdxHJtyx3Fq9vep1ha4ZRqkQpFnZeSLe63TRbd1Cuwt0YczMwwVrb2hhTA5gPuIEdwABrbZYxpjfQF8gAxlhrlxljLgHeBioDqUCktfZQAfQhIg75OfVnei7pyerdq2lfoz1v3PUGV4df7XRZQe+i31A1xgwF5gBnnjw7BYiy1rYAigGdjDFVgEFAM6AdMM4YEwr0B7Zn77sQiPJ9CyLihIysDOZunkudmXWI3xvPzDtnsrzrcgV7EZGb2w/sBu7N8b4REJf9egXQBrgJSLDWpltrU4AkoC7QHFh51r4i4ud2HNxB07lNeXzp49SsWJPNfTfT78Z+WoYpQi66LGOtXWSMqZ5jqJi11p39OhUoB4QDKTn2Od/4mbHzcrlcua86h7S0NK+P9VfqOTgUxZ5PZ51m7q65zPhmBmGlwph4y0Q6VOuA+5Ab16H811oUey5oBdWzNxdUs3K8DgOOAseyX//Z+Jmx84qIiPCiFM8/Ct4e66/Uc3Aoaj1v+3UbPRb3YMsvW3johoeY1n4alcpW8uk5ilrPhSE/PScmJl5wmzd3hdxijGmd/bo9EA9sAloYY0obY8oBEXgutiYAHc7aV0T8yKnMU4xaN4pGbzRif+p+Fj24iH/f/2+fB7v4ljcz92eAGGNMCOAC3rfWZhpjovGEd3FghLU2zRgzE1hgjFkPnAK6+qpwESl4W3/ZSuTiSLb+upWudboSfUc0FctUdLosyYVchbu1dg9wS/brb4FW59knBog5a+wE8EC+qxSRQpWZlcnkDZOJWhtFhUsqsPihxXT6Wyeny5I80JeYROR/fHPoG/os7UPCjwm637ofU7iLCOB5lunY+LGMXz+esNAwfcvUzyncRYQtB7bQfXF3dhzcQbe63ZjSdooumPo5hbtIEDudeZrx68cz+rPRVCpTiWVdlnFnzTudLkt8QOEuEqR2HtxJ5OJIEg8k0rVOV6a1n0aFSyo4XZb4iMJdJMicud96VGwU4aHhvP/A+9xX6z6nyxIfU7iLBJGk5CQiF0fy+Y+f0/lvnZndcTaVy1Z2uiwpAAp3kSCQ5c7itS9eY8TaEYSUCNEnYYKAwl0kwB1IPUCPJT1YvXs1HWt2ZNads7gq/Cqny5ICpnAXCVDpGelM/WIqY+PHkpmVyeyOs+ndsLdm60FC4S4SgLYc2MKjHz7KzkM7udvczeS2k6lRoYbTZUkhUriLBJATp08wat0oJm+YTOWylfW59SCmcBcJEBt+3MCjHz7K7t9206t+Lya2najPrQcxhbuInzt84jDj4sfx6sZXqRZejdjIWFpXb+10WeIwhbuIH1vz/Rq6LOpC8slketXvxeR2kwkPDXe6LCkCFO4ifig9I50Ra0cwZcMUalWqxdrua6lzRR2ny5IiROEu4me2HNhC5OJIth/cTr9G/ZjUdhJlQ8o6XZYUMQp3ET+RfDKZqLVRvJH4BpXK6g6O8ucU7iJFnNvt5l87/sWQT4bw6/Ff6duoLy/d9pI+CSN/SuEuUoR9e+RbBq0YxKrdq7jxyhtZ2mUpDas2dLos8QNehbsxphSwAKgOZAK9gQxgPuAGdgADrLVZxpjeQN/s7WOstcvyX7ZIYMtyZ/H2d28z9YOphJYMZUrbKQy6eRAlipdwujTxE8W9PK4DUNJa2xQYDYwFpgBR1toWQDGgkzGmCjAIaAa0A8YZY0LzX7ZI4NqdvJvb37qdl7e8TOvqrdn5xE4GNxmsYJc88XZZ5lugpDGmOBAOnAZuAeKyt68A2uKZ1SdYa9OBdGNMElAX+DJfVYsEILfbzdwtcxn48UBKlSjF6BtHE9UhSjf6Eq94G+7H8SzJ7AIuBzoCLa217uztqUA5PMGfkuO4M+MiksOh3w/Rf3l/FrkWcft1t7Og8wKO/nRUwS5e8zbcBwOrrLXDjTHVgLVASI7tYcBR4Fj267PHz+FyubwqJC0tzetj/ZV6Dixr9q/hxa9eJPV0Kk/XeZqepidHfzoa0D1fiHr2HW/D/Tc8SzEAyUApYIsxprW1dh3QHogFNgFjjTGlgVAgAs/F1nNERER4VYjL5fL6WH+lngPDbyd/46lVT7Fw60IaVGnAwnsWUrty7T+2B2LPF6Oe8yYxMfGC27wN96nAPGNMPJ4Z+z+Br4AYY0wI4ALet9ZmGmOigXg8F29HWGvTvDynSMBY9M0i+izrQ0paCs+3fJ4RLT2PvxPxFa/C3Vp7HHjwPJtanWffGCDGm/OIBJrfT/3O4FWDidkcQ+MrGxNzVwz1qtRzuiwJQPoSk0ghWZW0iv7L+7Pn6B6ea/Yco24dpdm6FBiFu0gBO/j7QQavGsy729+lZsWaxEbG0qr6Of/JFfEphbtIAVr0zSL6LuvLsfRjPN/yeYa3GE7pkqWdLkuCgMJdpAD8evxXBq0cxHs736NR1UYsvGchtSrVcrosCSIKdxEfW5m0ksjFkaSkpfDSrS8xrNkwSpUo5XRZEmQU7iI+cuTEEYatGcbcLXOpU7kOa7uv5YbKNzhdlgQphbuID/xr+7/ov7w/x08dZ2jToYy6dZTW1sVRCneRfEg+mcyzq5/lza/fpFm1ZszqOOt/vmUq4hSFu4iXYn+I5dEPH+XX339lePPhjGo9SmvrUmQo3EXyKDU9lZGxI4neGM31Fa9nY5eNejqSFDkKd5E8+MD1Af2X9+fQ74d4ovETTGgzgbIhZZ0uS+QcCneRXDh+6jj//PSfTNs07Y9nmd501U1OlyVyQQp3kYvY8OMGun3YjR9++4Enb36SCW0mEFpST4uUok3hLnIB6RnpjI0fy9j4sVQLr8a6HutoeU1Lp8sSyRWFu8h5LLVLGbhiIPtS9hFZL5Lo9tGEh4Y7XZZIrhV3ugCRoiQtI41nVz/L3f++m/Kly/PJo58wv/N8Bbv4Hc3cRbJt/WUrj374KNsPbqf/jf2Z0m6KvmUqfkvhLkEvPSOd8evHMyZ+DBUvqcjyrsvpcH0Hp8sSyReFuwS1DT9uoMeSHnx75Fu61O7CtPbTqFimotNlieSbwl2C0sHfDzJ8zXDmfT2Pa8pdw8pHVtKuRjunyxLxGYW7BJ2ldim9PupFSloKQ5oOYWTLkYSFhjldlohPeR3uxpjhwN1ACDADiAPmA25gBzDAWptljOkN9AUygDHW2mX5LVrEGylpKQz9ZChvbH6D+lXqE9cjTk9HkoDl1UchjTGtgaZAM6AVUA2YAkRZa1sAxYBOxpgqwKDs/doB44wx+mqfFLrl3y6n1oxazNkyh2ebPMsXj32hYJeA5u3MvR2wHfgQCAeGAL3xzN4BVgBtgUwgwVqbDqQbY5KAusCX+SlaJLd+Of4Lwz8dzvyv51O7cm0WP7SYxlc1droskQLnbbhfDlwDdASuBT4Ciltr3dnbU4FyeII/JcdxZ8bP4XK5vCokLS3N62P9lXq+uCx3Fu/tfo9J2yZxKvMUvf/WmwE3DCDkWAiuY/7xe6efc3AoqJ69DfcjwC5r7SnAGmPS8CzNnBEGHAWOZb8+e/wcERERXhXicrm8PtZfqec/t/foXh776DE+/eFT2lzXhhkdZnB9xesLuELf0885OOSn58TExAtu8/b2A+uBO4wxxYwxVwJlgU+z1+IB2gPxwCaghTGmtDGmHBCB52KriM+53W5mfzWb2jNrs3H/RmbdOYvV3Vb7ZbCL5JdXM3dr7TJjTEs84V0cGAD8AMQYY0IAF/C+tTbTGBONJ+iLAyOstWm+KV3k/+1L2cfjHz3OJ99/wm3X3sbcu+dS/bLqTpcl4hivPwpprR16nuFW59kvBojx9jwif8btdjNn8xyeWf0MWe4sZnSYQd8b+1K8mO6JJ8FNX2ISv/Vjyo/0XtqbVbtXcWv1W5l791yuLX+t02WJFAkKd/E7brebeVvm8fTqp8nIyuD1Dq/T78Z+mq2L5KBwF7/y07Gf6LO0DyuSVtDqmlbM6zSP68pf53RZIkWOwl38gtvt5p1t7zDg4wGczjpN9B3RDLhpgGbrIhegcJci7/CJwwzeMJjVP62mydVNWHjPQmpUqOF0WSJFmsJdiiy3282/d/ybwasGc+TkEcb9fRxDmg6hRPESTpcmUuQp3KVIOn7qOAM/HsiCrQtoVLURs5rNonOTzk6XJeI3tGApRc6GHzdQb1Y9Fm5dyAutXmDj4xsxlxmnyxLxK5q5S5Fx8vRJXlj3ApM3TKZaeDXiesTR4poWTpcl4pcU7uI4t9vNv3b8i+GfDmdfyj76NOzDxLYTCQ8Nd7o0Eb+lcBdHJZ9Mps/SPixyLaJBlQYs6LyA1tVbO12WiN9TuIsj3G43c7fMZegnQ0k9lcorbV7hmabP6HPrIj6icJdC98NvPzB41WCW2CXcWv1WXrvjNepcUcfpskQCisJdCs3J0ycZFTeKqV9MpUSxEky6fRKDmwzWbF2kACjcpVBs/GkjPZb0YNfhXUTWi2TsbWO5Kvwqp8sSCVgKdylQaRlpvBD7ApM2TOLKsCtZ1W0Vbf/a1umyRAKewl0KzJf7vyRycSSuwy4ea/AYk9tOplzp8z4fXUR8TOEuPpeekc6L617klc9f4cqwK1nxyAruqHGH02WJBBWFu/hU4s+JRC6OZOehnfSq34sp7aZoti7iAIW7+ERGVgbj4scx+rPRVC5bmY+7fkz769s7XZZI0FK4S77tTt5Ntw+78cVPX9Cldhde7/A65S8p73RZIkEtX+FujKkMJAK3AxnAfMAN7AAGWGuzjDG9gb7Z28dYa5flq2IpMk5nnua1ja/x4roXKVWiFO/c+w5d63R1uiwRIR+3/DXGlAJmAyezh6YAUdbaFkAxoJMxpgowCGgGtAPGGWNC81eyFAVf7v+S+rPrM+STIdx27W1s67dNwS5ShOTnq4GTgFnAz9nvGwFx2a9XAG2Am4AEa226tTYFSALq5uOc4rBj6cd4/KPHuXnOzRxLP8aSh5ew5OElVCtXzenSRCQHr5ZljDE9gEPW2lXGmOHZw8Wste7s16lAOSAcSMlx6Jnxc7hcLm9KIS0tzetj/ZVTPX958Euivoxi/4n99DQ96RvRl7CsMHbt2lXg59bPOTioZ9/xds29F+A2xrQB6gMLgco5tocBR4Fj2a/PHj9HRESEV4W4XC6vj/VXhd3zgdQDDF0zlLe3vc115a/jswc/o9lfmhXa+UE/52ChnvMmMTHxgtu8Cndrbcszr40x64B+wERjTGtr7TqgPRALbALGGmNKA6FABJ6LreIn4vfG8+D7D3I07SjDmw8nqmUUZUqVcbosEbkIX34U8hkgxhgTAriA9621mcaYaCAez/r+CGttmg/PKQUkJS2FEWtHMOPLGdSoUIM1j67hhso3OF2WiORSvsPdWts6x9tW59keA8Tk9zxSeOL2xNF9cXd+OvYTA28ayJjbxuiRdyJ+Rl9ikj8cSz/G8DXDmfnVTP5a4a8k9ErglqtvcbosEfGCwl0AWPHdCvou68v+1P3846Z/8PLfX6ZsSFmnyxIRLyncg9yRE0cYvGowb217i1qVavH5A59z89U3O12WiOSTwj1Iud1u3tn+Ds+sfobkk8lEtYgiqmUUoSX1BWKRQKBwD0I7D+5kwMcDiNsbR+MrG7O622rqVanndFki4kMK9yCSnpHOqLhRTPx8IuGh4czuOJvHGz6uB1SLBCCFe5DI+RCNnvV78srtr3B5mcudLktECojCPcD9fup3xq0fx/j147ni0itY3nU5Ha7v4HRZIlLAFO4BbNE3ixi0chA/p/5Mt7rdiL4jWg/REAkSCvcAtOfoHp5a+RRL7BIaVm3Ie/e/V+g3+hIRZyncA0hmViaTN0zmhXUvULxYccb/fTxPN3maUiVKOV2aiBQyhXuA+P6374lcHMn6fevp/LfORN8RrQdoiAQxhbufy3JnMWXDFEbGjiSkRAgLOy+kW91uFCtWzOnSRMRBCnc/lpScRP/l/Vnz/Ro6mU5M7zCdq8OvdrosESkCFO5+6HTWaUbHjWZs/FhCS4Qyu+Nsejfsrdm6iPxB4e5ndh3eRbe13dievJ2Haz/MlLZTqBpW1emyRKSIUbj7iZOnTzJ5w2RGx42mTMky/PeB/3J/rfudLktEiiiFux9Yv289PZf0JCk5iQdveJCBfx1Ii1otnC5LRIow3TGqCDtx+gSDVw6m5ZstyczKZFW3Vfzn/v9weWndE0ZE/pxm7kVU/N54en3Ui6TkJAY0HsD4NuO5NORSp8sSET+hcC9ifj/1OyPWjiB6YzTVL6tObGQsrau3drosEfEzXoW7MaYUMA+oDoQCY4BvgPmAG9gBDLDWZhljegN9gQxgjLV2Wf7LDkzxe+PpuaQnu3/bzcDGAxnXZpxm6yLiFW/X3LsBR6y1LYD2wHRgChCVPVYM6GSMqQIMApoB7YBxxhg9x+0sJ0+f5KmVT9Fqfiuy3FnERsYyrcM0BbuIeM3bZZn/Au/neJ8BNAList+vANoCmUCCtTYdSDfGJAF1gS+9PG/ASdiXQO+lvXEddmltXUR8xqtwt9YeBzDGhOEJ+ShgkrXWnb1LKlAOCAdSchx6ZvwcLpfLm1JIS0vz+lgnJaUkMXbLWDYe3EjVMlWJaRlDsyrN+HH3jxc91l97zg/1HBzUs+94fUHVGFMN+BCYYa191xjzSo7NYcBR4Fj267PHzxEREeFVHS6Xy+tjnXA07Sij1o1i2qZphIWG8UqbV+jfuH+eZuv+1rMvqOfgoJ7zJjEx8YLbvL2gegWwGhhorf00e3iLMaa1tXYdnnX4WGATMNYYUxrPhdcIPBdbg9KHrg/ps6wPR04coXfD3oy5bQyVylZyuiwRCUDeztz/CZQHRhpjRmaPPQlEG2NCABfwvrU20xgTDcTjuXg7wlqblt+i/U1KWgrPrH6GuVvm0rBqQ1Z3W02Dqg2cLktEApi3a+5P4gnzs7U6z74xQIw35/F3brebmV/NZFTcKA6fOMzw5sN5sfWLhJQIcbo0EQlw+hJTAbGHLQNXDGTN92to/pfmLOuyjMZXNXa6LBEJEgp3H8vMymTi5xMZGTuSMqXKMPPOmfRt1Ff3WheRQqVw96G4PXE8v+55Ptv7GffXup/p7adzxaVXOF2WiAQhhbsPHEg9wMAVA/nA9QGXl7mcOXfNoVeDXpqti4hjFO754Ha7eXf7uwxcMZCTp0/y8m0vM7jJYEqXLO10aSIS5BTuXko+mcwTy5/gPzv/Q9NqTXmz05vUrFjT6bJERACFe54dOXGE6I3RTP9yOsfSjzHm1jE81/w5ShQv4XRpIiJ/ULjnktvt5p3t7/DUyqc4cvIInUwnXmz9IvWr1He6NBGRcyjcc2Hbr9t4dvWzfPL9JzS5ugmzO86mzhV1nC5LROSCFO5/IiMrg+mbpjNszTDKlCrDq+1eZeBNA7UEIyJFnsL9PDKyMpizeQ5TNkzhu+Tv6FizI/PunqebfImI31C4n+Wrn7+i99LefP3L1zSs2pBFDy7inr/do8+si4hfUbhnS89IZ3TcaCYkTKBy2cr894H/cl/EfQp1EfFLQR/umVmZxO6J5elVT7P94HZ61u/J1HZTKVf6vA+MEhHxC0Eb7hlZGby97W1GxY1iz9E9VLm0Cku7LKVjzY5OlyYikm9BGe4J+xJ44uMn2PbrNhpWbciENhPoWLMjZUqVcbo0ERGfCKpw33V4Fz5ym30AAAUKSURBVOPXj2fB1gVUC6/G+w+8z70R92pdXUQCTlCE+6b9m3jps5dY/u1yShQvwbBmwxjZciRlQ8o6XZqISIEI6HDffGAzQz4Zwtof1nJ5mcsZ0WIEfW/sy9XhVztdmohIgQrIcP859WdGx43mjcQ3qFS2EhPaTKD/jf0JCw1zujQRkUJR4OFujCkOzADqAenA49bapII4176UfUxYP4G5W+aSkZXBkzc/yYutX9THGkUk6BTGzL0zUNpa28QYcwswGejkyxP88NsPvBz/Mgu2LgCgR/0ePNf8Oa4rf50vTyMi4jcKI9ybAysBrLVfGGNu9NUvnOXOYvqO6cz7YB5ut5s+jfowtNlQ/lLuL746hYiIXyqMcA8HUnK8zzTGlLTWZuT3F9726zZmfDODh254iEltJ+lCqYhItsII92NAziuZxc8X7C6XK8+/cCihxLWPo1JYJVL3p+Lan/dfwx+lpaV59fvlz9RzcFDPvlMY4Z4A3AW8l73mvv18O0VERHj3q7vycayfcrlc6jkIqOfgkJ+eExMTL7itMML9Q+B2Y8znQDGgZyGcU0QkqBV4uFtrs4B+BX0eERH5f8WdLkBERHxP4S4iEoAU7iIiAUjhLiISgBTuIiIBqJjb7Xa6BhITE50vQkTEDzVq1Oi8TxsqEuEuIiK+pWUZEZEApHAXEQlAfvskpsJ8CEhhM8aUAuYB1YFQYAzwDTAfcAM7gAHW2ixjTG+gL5ABjLHWLnOiZl8xxlQGEoHb8fQ0nwDu2RgzHLgbCMHz5zmOAO45+8/2Ajx/tjOB3gTwz9kYczMwwVrb2hhTg1z2aYy5BHgbqAykApHW2kN5Obc/z9z/eAgI8Byeh4AEim7AEWttC6A9MB2YAkRljxUDOhljqgCDgGZAO2CcMSbUoZrzLfsv/mzgZPZQQPdsjGkNNMXTSyugGgHeM9ABKGmtbQqMBsYSoD0bY4YCc4DS2UN56bM/sD1734VAVF7P78/h/j8PAQF89hCQIuC/wMgc7zOARnhmdQArgDbATUCCtTbdWpsCJAF1C7NQH5sEzAJ+zn4f6D23w3OX1A+BpcAyAr/nb4GS2f/zDgdOE7g97wbuzfE+L33+kW859s0Tfw738z4ExKlifMlae9xam2qMCQPex/OvdjFr7ZmPNqUC5Tj39+DMuN8xxvQADllrV+UYDuiegcvxTEoewHNzvXfwPO8gkHs+jmdJZhcQA0QToD9na+0iPP94nZGXPnOOe9W7P4d7rh4C4q+MMdWAWOAta+27QFaOzWHAUc79PTgz7o964bk19DqgPp7/ilbOsT0Qez4CrLLWnrLWWiCN//1LHIg9D8bTc00818sW4LnecEYg9nxGXv4O5xz3qnd/DvcEPOt3/NlDQPyRMeYKYDUwzFo7L3t4S/YaLXjW4eOBTUALY0xpY0w5IALPhRq/Y61taa1tZa1tDXwNdAdWBHLPwHrgDmNMMWPMlUBZ4NMA7/k3/n9GmgyUIsD/bOeQlz7/yLcc++aJPy9jBPJDQP4JlAdGGmPOrL0/CUQbY0IAF/C+tTbTGBON5wdfHBhhrU1zpOKC8QwQE6g9Z38qoiWev+DFgQHADwRwz8BUYJ4xJh7PjP2fwFcEds9n5PrPszFmJrDAGLMeOAV0zevJ9A1VEZEA5M/LMiIicgEKdxGRAKRwFxEJQAp3EZEApHAXEQlACncRkQCkcBcRCUAKdxGRAPR/gcD+ocA39A0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "bar() missing 1 required positional argument: 'height'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-145-a4b1fcdefdb8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m9\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'cyan'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'A'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbottom\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0ma1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'green'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'B'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbottom\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0ma1\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0ma2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'red'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'C'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma4\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbottom\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0ma1\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0ma2\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0ma3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcolor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'yellow'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'D'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: bar() missing 1 required positional argument: 'height'"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 648x432 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#stacked bar plot attempt\n",
    "figure(figsize=(9,6))\n",
    "plt.bar(a1, color='cyan', label='A')\n",
    "plt.bar(a2, bottom=a1, color='green', label='B')\n",
    "plt.bar(a3, bottom=a1+a2, color='red', label='C')\n",
    "plt.bar(a4, bottom=a1+a2+a3, color='yellow', label='D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7.067241188649765,\n",
       " 126.82720352595265,\n",
       " 26.606157845736153,\n",
       " 2.660647692110857,\n",
       " 728.1099290535948)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rewardSum[0], rewardSum[1], rewardSum[2], rewardSum[3], rewardSum[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "728.1099290535948"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rewardSum[9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}