CMAB - Choosing Top 2 Arms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "%matplotlib inline\n",
    "from matplotlib.pyplot import figure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1., 1., 1.])"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.ones(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Probability</th>\n",
       "      <th>Arm1</th>\n",
       "      <th>Arm2</th>\n",
       "      <th>Arm3</th>\n",
       "      <th>Arm4</th>\n",
       "      <th>Arm5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.2</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Probability  Arm1  Arm2  Arm3  Arm4  Arm5\n",
       "0          0.2   0.0   0.1   0.2   0.2   0.0\n",
       "1          0.2   0.1   0.3   0.5   0.5   0.1\n",
       "2          0.2   0.1   0.4   0.6   0.6   0.1\n",
       "3          0.2   0.3   0.4   0.8   0.8   0.3\n",
       "4          0.2   0.4   0.5   1.0   1.0   0.4"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data1 = {'Probability': np.ones(5)*0.2,\n",
    "        'Arm1': [0, 0.1, 0.1, 0.3, 0.4],\n",
    "        'Arm2': [0.1, 0.3, 0.4, 0.4, 0.5],\n",
    "        'Arm3': [0.2, 0.5, 0.6, 0.8, 1],\n",
    "        'Arm4': [0.2, 0.5, 0.6, 0.8, 1], #arm4 has same prob dist as arm3\n",
    "        'Arm5': [0, 0.1, 0.1, 0.3, 0.4], #arm5 has same prob dist as arm1, but with window size<5, can't observe this relationship\n",
    "       }\n",
    "\n",
    "df1 = pd.DataFrame(data1)\n",
    "df1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Distribution of Rewards for Each Arm')"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEPCAYAAACjjWTcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAccUlEQVR4nO3de5hU1Znv8W83F5tOg4gobZxcjOhreYmJoIJHE5MjDPSJE9TEJzHOIOh4SSJxMDpMjuOTyTFxEmUythkvGB2JSfTEUTJjxFYnaqIimdBRQW1eaS8Tjh0N0IKQ5taX88faLUVZ3V3V1K6me/0+z8MD+/7uRdWvVq2q2ruiq6sLEREZ2ioHugAREUmfwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJALDB7oAKZyZfRh4BViVzKoEtgD/7O4/S9b5FtDs7j/qZT9XA8+7+7/nWfbu9mbWBRzg7uuLqPF44Hx3v9jMJgML3P1zhW7fH2Y2DLgfyAD17v6DrGXnATcAryWzKoAxwJPAhe6+Lc3aemNmW4Cj3f31AtefDtwGvAV80t239uOYpwIPAZ6zaL27n1bs/pJ9vg58zt1XFLDuMcBKwuPiu/05nvSPwn7w2eruH+ueMLMPAb80sw53v8/dry5gH58GXsq3oMDte3MU8GfJvlYAqQZ94mDgz4H3uXtHnuVPuvtnuifMrAp4CpgN3FqG+krlC8Bt7n7NHu7nlezHUJl9GfgJ8FUzW+ju7QNUR3QU9oOcu/930lO/ArjPzO4EXnD3683sH4AzgB3ABuA84ExgMnCdmXUAnwXGAYcCvwAmdG+fHOLbSW+9ErjK3X+R9JY/1x2g3dPAJcC3gH3N7F+BxcAP3P1oM9sX+BfgY0AXoXf5DXdvN7NtwD8C04GDgO+5+82552pmpwDXAdXJOV0FPA00ACOARjM7y91f6aPZ9gf2BVqT/R4M/AD4YLKfe9z9O2b2c+ABd7/dzKYCy4BD3f1VM7sKGA38E+EFYwJQC/w3cLa7/zHp8f4G+CjwDWA9cGNy/r9N2hQzqwH+FTgM6AQagYvcvTPr3K8AZgFbk7b8RnLs/wl0JMf5G3ffnHtcd1/SR3u8y8wm9HI+hyfLDkzqvMbd/2+y6UVmdkuy7C53/9959j0a+BJwIuFx8DngnmTZN4GpwPuB54FmwmPyYMJjohF4nPACfQhwpbvfXeh5icbsh4rngWOyZ5jZB4DLgOPdfTLwCHCiu/8LsAK4IisEqt39KHf/2zz7ftXdjwPOBRab2QE9FeHua4GrCT3pOTmL6wkvOMcQXmyOBb6eLNuHMIxwEiEAvp/0vrPPZ3/g34CvuftHCU/6HwPjgTqSdzw9BP0pZvacma02s3XAz4Dr3f3eZPldwB3uPgk4ATjNzM4mDA3NTNaZAbwJdA91/AVwH6G3/Yy7TwU+ArQBf5l17BfcPQM8CNwLXO7uHycE16hknTOA0Ulv+/hk3kdy2vY64D+A77v7FYQXuvcn7Xgs4bl8Xe5xewj6Q5P2yP7THc69nc89wL3ufhShzb9jZmOSZduSx9kJwOXJ4y/XXwIvu3sToSPwNznLPwR83N3PTaZPTtrmuOR4R7r7J4CvAv+QZ//SC4X90NBFeFJme4PwIvA7M7seeM7df97D9k/1su9bANz9BcLQz9R+1jiT0MvvcvftyX5nZi3v/vzgd4Twf1/O9icSPkv4TVLPi4Re/akFHPvJJEiPJPSs9ycEL2b2PuCTwP8xs+eA5YQe/seAB4BTzWw4YZjoGmCamb2f0IP9rbvfACwzs/nATcDRQE32sZO/jwF2uvsvk/rvBjYny54CjjKzJ4AFhM9gmvs4p5nALe6+M3kHcCO7t+eT+TcDkmGcnD/fTurKez5mNo7wovLDZL217n6ou7+T7POnyfw3CZ8pHJjnuBcTQh7CC/Wk5B1Tt+U5wzr/6e6bks8mWgjv4CB8bjWul/OTPBT2Q8Px7PrQFoAkAD5JGLrZQOgtf6+H7bf0su/sMfBKYCfhxaUia/7IAmqsTLbLnh6RNb01qbt7nez9AwzL2T7fPnrl7p3u/i3gdeDOrP1WACd1Bx8wBfiOu78NPAecTvhQ90fAKYThlCXu3mVm3yUMXa0DFhHeQWXXnt22uefUntT1GjARuDY5zn+a2el9nE5ue+S2RW//pz3q5Xy6Q7gra10zs+53JzuzdpP7+OgegjsauDIZZnqGMBSX3bvPrXl7zvROpN8U9oNcMo7698DCnPnHAi8ATe5+LfB9dg0RtFN4SJ6X7O84QiD9hhAER5tZlZmNYPcPYXva98OED+UqzGwf4ELg0QJrgBAOR5jZCUk9RwGfAJ4oYh/dvgJMN7PPJj3T5cD8ZL9jCe8YPpusez/wHeCX7r4ZeJnQ+74vWf7nhJ74XcAfgWmEIM61Eqgws7rkOH8B7Jf8+xLCmP0jyVDaw4Shi940AJeY2Qgzq0zOqZj27Ene80naqZEwfNY9TPg04bOPQlxCGMv/gLt/2N0/DHwGONPMPliCuqUPCvvBZ1TWOOvvCD3Uv3P3B7NXcvfnCWPTK8xsBTCXJNAIY7/XmtnsAo73ETN7lvD2/Qvu3kro7f0KWA38mvAZQLflyTb35+xnHuGt/arkjwPfLvCcSb7++XngRjNbRRg2mOPuLxe6j6x9vQJ8l12fDZwDTEn2+xvgbnf/SbL6zwFjV5A+THgxW5ZMfwu43sxWEtr1KcKLYu4xdxLeEXQPF51JCFMI7xiGAS+ZWSMhQOv7OI1rCJ8hPAc0JTV9rcAmyDdm/5yZHdjH+ZwDnG1mzxOGuC5Ihm16lXzOcya7f6aAuz9GeBG/tMC6ZQ9U6BLHIiJDn3r2IiIRUNiLiERAYS8iEgGFvYhIBBT2IiIR2GuvjdPY2KivCYmI9MOkSZNyf8C394Y9wKRJkwa6BBGRQaWxsTHvfA3jiIhEQGEvIhIBhb2ISAQU9iIiEVDYi4hEILWwN7MTk5sx5M4/3cx+a2bPmNlfp3V8ERHZJZWwN7MrCZfEzb213AjCddWnE26scaGZ1aZRg4iI7JLW9+xfIVy/+q6c+RnCreXeBjCzpwh3/rkXkTJraGhg6dKlRW3T2toKwLhxxd0Vr66ujhkzZhS1TTmpLYa+VMLe3e8zsw/nWTQG2JQ1vZle7nTT1NRU4spEdmlpaaGtLffWvb1bt24dAFVVVX2s+d5j7c2PZ7XF0FfuX9C+A4zOmh4NbOxp5Uwmk3pBEq9MJsPcuXOL2mbevHkA1Nf3dSOpwUVtMXT09Avacod9E3BYcqf6LYR7iF5f5hpERKJTlrA3s3OAGndfZGbzCffxrATucPc3ylGDiEjMUgt7d38dmJL8+6dZ8x8g3KxYRETKRD+qEhGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgMT2OnZlYJ3AQcC2wHLnD35qzlXwIuBzqAO9z95jTqEBGRIK2e/Sygyt2nAguAhTnLrwdOA/4HcLmZ7ZdSHSIiQko9e+BkoAHA3Zeb2eSc5SuBfYF2oALoSqkOiUB9fT3Nzc19r1gCa9asAWDevHllOd7EiROLOpbaQnqSVtiPATZlTXeY2XB3b0+mXwAagT8B97v7xnw7aWpqSqk8GUpWrlyJv+YwtgwHS94LP/vGs+kfayO0tbUV9TxYuXIlv3/5ZWpTLKvbqOTvjc89l/qx3qT4tpDdpRX27wCjs6Yru4PezD4K/C/gEGAL8GMz+7y735u7k0wmk1J5MpRUV1fDWOg8tXOgSympyicqqa6uLup5UF1dTS1wPhXpFTYAbqer6LaIVWNjY975aY3ZPw3UAZjZFGBV1rJNwFZgq7t3AH8ENGYvIpKitHr2S4BpZraMMCY/x8zOAWrcfZGZ3Qo8ZWY7gFeAO1OqQ0RESCns3b0TuDhn9uqs5bcAt6RxbBEReS/9qEpEJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYnA8DR2amaVwE3AscB24AJ3b85afjzwT0AF8CZwrrtvS6MWERFJr2c/C6hy96nAAmBh9wIzqwBuA+a4+8lAA/ChlOoQERFS6tkD3SGOuy83s8lZyw4HNgCXmdkxwIPu7inVIRFobW2FjVD5xBAbldwIraNai9qktbWVt4Db6UqnpgHyB6Cztbi2kN2lFfZjgE1Z0x1mNtzd24HxwEnApcAa4Bdm1ujuv8zdSVNTU0rlyVCyY8eOgS4hNTt27CjqeaC2kJ6kFfbvAKOzpiuToIfQq29295cAzKwBmAS8J+wzmUxK5clQUltbyx86/kDnqZ0DXUpJVT5RSW1tbVHPg9raWqrefJPzqUixsvK7nS7GFtkWsWpsbMw7P633vU8DdQBmNgVYlbXsVaDGzCYm06cAL6ZUh4iIkF7PfgkwzcyWEb5xM8fMzgFq3H2RmZ0P/DT5sHaZuz+YUh0iIkJKYe/uncDFObNXZy1/DDghjWOLiMh7DbGvL4iISD4KexGRCCjsRUQioLAXEYmAwl5EJAJ9fhvHzIYB5wEfBB4HXnD39SnXJSIiJVRIz/5WwoXKphN+FfujVCsSEZGSKyTsD3X3q4Gt7v4AsG/KNYmISIkVEvbDzWw8gJmNBobWBUhERCJQyC9oryJc6+YgYDlwWaoViYhIyfUZ9u7+K8DM7ABgvbsPrQtli4hEoJBv4zwOu+6EYGa4+6dTrUpEREqqkGGc7guaVRCuO39seuWIiEgaChnGyb5l4Gozm5tiPSIikoJChnEuzJo8iN3vQCWDTENDA0uXLi1qm9bk3p/jxo0r+nh1dXXMmDGj6O2KVq570G5L/q5K/1BsBA4ufrM3Kc89aLckf9ekfqRwTmPLcJyhrJBhnIOy/r0NODulWmQvtWHDBqB/YV8OEydO7HulElmzZg0Ahx18WPoHO7j4cytnW6xL2uLPDku/LcZS3nMbiiq6uvL3AMzs8J42cveXU6so0djY2DVp0qS0DyMFmDdvHgD19fUDXMnAU1vsorbYOzU2NjJp0qT33IS4t579rT3M7wL0bRwRkUGkx7B390/lm29mI9MrR0RE0lDIB7QXAfOBEYSvX+4EehziERGRvU8hX1/4a+BU4CFgDvBSmgWJiEjpFRL2G9z9D8Bod38C2Du/kiEiIj0qJOw3mtksoCsZ0jkg5ZpERKTECgn7A4CjgQWEsfpLUq1IRERKrpCwvxKoJYzZbwZeTbUiEREpuUKujbMCWGFm+wE3A2uAfdIuTERESqeQr16eQrjh+PHAvcDXU65JRERKrJBr41wG3AZcoBuXiIgMToUM45xVjkJERCQ9ZbgmrIiIDDSFvYhIBAoZsy+amVUCNxFuYbidMN7fnGe9RUCruy9Iow4REQnS6tnPAqrcfSrhx1gLc1dIfo17TErHFxGRLKn07IGTgQYAd19uZpOzF5rZVGAK4Zr5R6RUw7uG4q34Lr/8cl56qTzXpNu6dSsAM2fOLMvxjjzySBYufE//QKQsypkXZbttJ+mF/RhgU9Z0h5kNd/d2MzsI+CZwBn3c4rCpqakkxbS0tNDW1lbUNuvWrQOgqqr4m422tLSUrPaerF27lq1/2sI+w9L/Nuxwwk1vOrdtTv1Y2zsqWLt2bert11/dj6O9tb5yGqptUc68KEdWdEsr7N9h9xuTV7p7e/LvzwPjgaWEyzBUm9lqd78zdyeZTKYkxWQyGebOnVvUNnv7Lddqa2sZu20tV03e0vfKg8g1K2qoqq0t2f99qVVXVwOle2wOZkO1LQZ7XjQ2Nuadn1bYPw2cDvzMzKYAq7oXuHs9UA9gZucBR+QLehERKZ20wn4JMM3MlhHubjXHzM4Batx9UUrHFBGRHqQS9u7eCVycM3t1nvXuTOP4IiKyO/2oSkQkAgp7EZEIKOxFRCKgsBcRiYDCXkQkAgp7EZEIKOxFRCKgsBcRiYDCXkQkAgp7EZEIKOxFRCKgsBcRiYDCXkQkAgp7EZEIpHU9+1TU19fT3NxclmOtWbMG2HUHmrRNnDix6GP9fsswrllRU9Q2m3ZUsHF7+V7jx+7Tyb4jC7914u+3DOPwFOuReCgvdjeowr65uZlnV71EZ3XxNwEvVkVHaJrGV95M/ViVba1FbzNx4sR+HauttZXKDRv6tW1/jNh/f6qKuAnz4fT/3ESyNTc38+KqJsZWH5j6sSo79gHgjVfSf25tbPtjv7YbVGEP0Fk9jm1Hfmagyyipqpd+UfQ25epBiAxmY6sP5FNHfGGgyyipx1ff06/tNGYvIhIBhb2ISAQU9iIiEVDYi4hEQGEvIhIBhb2ISAQU9iIiEVDYi4hEQGEvIhIBhb2ISAQU9iIiEVDYi4hEQGEvIhIBhb2ISAQU9iIiEUjlevZmVgncBBwLbAcucPfmrOVfBC4DOoCVwJfdvTONWkREJL2e/Sygyt2nAguAhd0LzGwUcA3wKXc/CdgXGFp3IxER2cukdaeqk4EGAHdfbmaTs5ZtB05y97asGralVIdIjxoaGli6dGlR2/T3XqN1dXXMmDGjqG3KSW0x9KUV9mOATVnTHWY23N3bk+GatwDM7FKgBng0306ampp2m25ra8u32pDQ1tb2nvOVdLW0tBT9mKqpCTd4L3a7lpaWvfr/dyi2hfJid2mF/TvA6KzpSndv755IxvS/R7i/9Fnu3pVvJ5lMZrfp6urqZNdDT3V19XvOV9KVyWSYO3fuQJexVxiKbVFdXc3bbB3oMlLRW140NjbmnZ/WmP3TQB2AmU0BVuUsvxWoAmZlDeeIiEhK0urZLwGmmdkyoAKYY2bnEIZsVgDnA08Cj5kZwA3uviSlWkREopdK2Cfj8hfnzF6d9W99v19EpIwUuiIiEVDYi4hEQGEvIhIBhb2ISAQU9iIiEVDYi4hEQGEvIhIBhb2ISAQU9iIiEVDYi4hEQGEvIhIBhb30af369Vx66aVs2LBhoEsRkX5S2EufFi9ezMqVK1m8ePFAlyIi/aSwl16tX7+ehx56iK6uLh566CH17kUGqbSuZ5+K1tZWKje/RfWKMvQwOzvD35VleD3saKe1dWT6x+mHxYsX09UVbiTW2dnJ4sWLmT9//gBXJdK31tZW1m/+fyz5XX3qx+rs7ACgsnJY6sdq79jBqNaKorcbVGE/YcKEsvUst24NtzMbVVWOEB7JhAkTynCc4j366KPs3LkTgJ07d/LII48o7GVQKG9ehOfIyKp9Uj/WSIb3Ky8GVdgvXLiwbMeaN28eAPX16fcK9mbTpk1j6dKl7Ny5kxEjRjB9+vSBLkmkIMqL3WnMXno1e/ZsKirCW8bKykpmz549wBWJSH8o7KVX48ePZ+bMmVRUVDBz5kz233//gS5JRPphUA3jyMCYPXs2r7/+unr1IoOYwl76NH78eG688caBLkNE9oCGcUREIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJgMJeRCQCCnsRkQgo7EVEIqCwFxGJQCqXSzCzSuAm4FhgO3CBuzdnLT8duBpoB+5w99vSqENERIK0evazgCp3nwosAN69sLSZjQC+D0wHPglcaGa1KdUhIiKkdyG0k4EGAHdfbmaTs5ZlgGZ3fxvAzJ4CTgHuTakWGhoaWLp0aVHbrFmzBth1U4Ji1NXVMWPGjKK3E5GBV868KGdWpBX2Y4BNWdMdZjbc3dvzLNsM7JtvJ01NTSUppqWlhba2tqK2qampASh6u+7jlap2ESmvcuZFObMirbB/BxidNV2ZBH2+ZaOBjfl2kslkSlJMJpNh7ty5JdmXiAxtgz0vGhsb885Pa8z+aaAOwMymAKuyljUBh5nZODMbCXwCeCalOkREhPR69kuAaWa2DKgA5pjZOUCNuy8ys/nAw4QXmzvc/Y2U6hAREVIKe3fvBC7Omb06a/kDwANpHFtERN5LP6oSEYmAwl5EJAIKexGRCCjsRUQioLAXEYlAWl+9LImefhwgIiLFqejq6hroGkREJGUaxhERiYDCXkQkAnv1mH2azOxvgcuAQ9x9Wz/3UQ08Cpzv7qv7Wn9vtadtYWZfTLbvAFYCX05+RT3olKAtziLcw6ELWOTuPyxxiWVTiudIsp9FQKu7LyhZcWVWgsfFfOB8YF0y6yJ39xKW2KeYe/ZfAu4BvtCfjZNr9P8aOLSURQ2QfreFmY0CrgE+5e4nES5X/ZnSlldWe9IWw4B/BE4DpgJXmNn40pZXVnv0HAEws4uAY0pW0cDZ07Y4Dvgrdz81+VPWoIdIe/ZmdirwCnAL8GPgTjN7gvCqux9wN+GqnaOAg4AbgM8CRwNfd/d/B/YBzgDuKnP5JbWnbUG4xtFJ7t59Ie/hQL97gQOpFI8LM8u4e7uZHUi4COCWcp9HKZSoLaYCU4BbgSPKfAolU6K8mAT8XXJXvgfd/doyn0a0PfsLgB8mr67bzezEZP5P3f00wnDEaHevA74LXAKcCVwIzAFw96fdfW35Sy+5PWoLd+9097cAzOxSoIYwtDUYleJx0W5mZwLPE9757SzzOZTKHrWFmR0EfBP4StkrL709flwQ3hVcDHwaONnMyv7uN7qwN7P9CK/CXzOzBsKww1eTxdlvrZ5N/t4INLl7F/A2UFWuWtNWqrYws0ozux6YBpyVLB9USvm4cPf7gYOBkcBfpVx6yZWoLT4PjAeWEj7DOMfMzku/+tIqRVuYWQXwz+6+3t13AA8CHy/LCWSJcRjnXOB2d78C3v2Q9TVgPZD9oeKgC6x+KFVb3ApsB2YN1g9mKUFbmNkYwrDWdHffbmZ/ytl2sNjjtnD3eqA+2f484Ah3vzOletNUiufIGOAFM8sAfyL07u9Ip9yeRdezJ7wle3ecPRlrvg84bMAqGjh73BZmdhzhWwbHAI+Z2RNmdkapCy2DPW4Ld38H+AnwazN7ihAAPy5xneWg58gupXhcbAK+ATwOPAm86O7F3dG8BPQLWhGRCMTYsxcRiY7CXkQkAgp7EZEIKOxFRCKgsBcRiYDCXqQPZjbDzC7sZfk3zeziPPPfTLcykcLF+KMqkaK4e8NA1yCypxT2EhUzux+4wd1/ZWbHA98jXNBqLOHn/be5+815LnR1mLsvMLNrgcnAaMLP4ruvfXKGmZ0NVAPz3P2/so55DOHXpBXABmBu8kMbkbLRMI7E5jZgdvLv8wi/arzH3acTLs08P2vd7AtddV8O4W13nwacBEwxs4OTdV9z908Tfk18S55jfsXdTyVcK+bKUp+USF/Us5fYPAxcZ2bjgFOAmcC1yZUq3wFGZK2be83xrcCBZnY34dLFNVnr/xrA3V9MLmObLQPcZGYk679cutMRKYx69hKV5EJt9wI3Az8HLgeecfdzk/kVWavnXsRsJvABd/8i4Vono7LWPwHeHbL5fe5hSW5cQejVP1iq8xEplHr2EqM7gFcJF7M6BLjZzL5EGE9vN7N9etjuv4C/N7PlhKt8vgq8P1l2iJk9RripzUU5210C/Ci5kxWEoR6RstKF0EREIqBhHBGRCCjsRUQioLAXEYmAwl5EJAIKexGRCCjsRUQioLAXEYmAwl5EJAL/HwKM0wcImpdGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax = sns.boxplot(x=\"variable\", y=\"value\", data=pd.melt(df1[['Arm1','Arm2','Arm3','Arm4','Arm5']]))\n",
    "plt.title(\"Distribution of Rewards for Each Arm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.88"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#e.g. trying arms with different types of distributions\n",
    "arr = np.arange(0.01, 1.01, 0.01) #probability dist values\n",
    "np.random.choice(arr)     #one random value within probability dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1 , 0.11,\n",
       "       0.12, 0.13, 0.14, 0.15, 0.16, 0.17, 0.18, 0.19, 0.2 , 0.21, 0.22,\n",
       "       0.23, 0.24, 0.25, 0.26, 0.27, 0.28, 0.29, 0.3 , 0.31, 0.32, 0.33,\n",
       "       0.34, 0.35, 0.36, 0.37, 0.38, 0.39, 0.4 , 0.41, 0.42, 0.43, 0.44,\n",
       "       0.45, 0.46, 0.47, 0.48, 0.49, 0.5 , 0.51, 0.52, 0.53, 0.54, 0.55,\n",
       "       0.56, 0.57, 0.58, 0.59, 0.6 , 0.61, 0.62, 0.63, 0.64, 0.65, 0.66,\n",
       "       0.67, 0.68, 0.69, 0.7 , 0.71, 0.72, 0.73, 0.74, 0.75, 0.76, 0.77,\n",
       "       0.78, 0.79, 0.8 , 0.81, 0.82, 0.83, 0.84, 0.85, 0.86, 0.87, 0.88,\n",
       "       0.89, 0.9 , 0.91, 0.92, 0.93, 0.94, 0.95, 0.96, 0.97, 0.98, 0.99,\n",
       "       1.  ])"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.arange(0.01, 1.01, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "arm1R = [] #arm1 rewards list\n",
    "arm2R = [] #arm2 rewards list\n",
    "arm3R = [] #arm3 rewards list\n",
    "arm4R = [] #arm4 rewards list\n",
    "arm5R = [] #arm5 rewards list\n",
    "\n",
    "#arm1 dist\n",
    "for i in range(1000):\n",
    "    if np.random.choice(arr) < 0.21:  \n",
    "        arm1 = 0    #20% chance\n",
    "    elif np.random.choice(arr) < 0.41:\n",
    "        arm1 = 0.1    #20% chance\n",
    "    elif np.random.choice(arr) < 0.61:\n",
    "        arm1 = 0.1    #20% chance\n",
    "    elif np.random.choice(arr) < 0.81:\n",
    "        arm1 = 0.3    #20% chance\n",
    "    elif np.random.choice(arr) < 1.01:  \n",
    "        arm1 = 0.4    #20% chance\n",
    "    arm1R = np.append(arm1R, arm1)\n",
    "#print(arm1R)\n",
    "\n",
    "#arm2 dist\n",
    "for i in range(1000):\n",
    "    if np.random.choice(arr) < 0.21:  \n",
    "        arm2 = 0.1    #20% chance\n",
    "    elif np.random.choice(arr) < 0.41:\n",
    "        arm2 = 0.3    #20% chance\n",
    "    elif np.random.choice(arr) < 0.61:\n",
    "        arm2 = 0.4    #20% chance\n",
    "    elif np.random.choice(arr) < 0.81:\n",
    "        arm2 = 0.4    #20% chance\n",
    "    elif np.random.choice(arr) < 1.01:  \n",
    "        arm2 = 0.5    #20% chance\n",
    "    arm2R = np.append(arm2R, arm2)\n",
    "\n",
    "#arm3 dist\n",
    "for i in range(1000):\n",
    "    if np.random.choice(arr) < 0.21:  \n",
    "        arm3 = 0.2    #20% chance\n",
    "    elif np.random.choice(arr) < 0.41:\n",
    "        arm3 = 0.5    #20% chance\n",
    "    elif np.random.choice(arr) < 0.61:\n",
    "        arm3 = 0.6    #20% chance\n",
    "    elif np.random.choice(arr) < 0.81:\n",
    "        arm3 = 0.8    #20% chance\n",
    "    elif np.random.choice(arr) < 1.01:  \n",
    "        arm3 = 1    #20% chance\n",
    "    arm3R = np.append(arm3R, arm3)\n",
    "\n",
    "#arm4 dist\n",
    "for i in range(1000):\n",
    "    if np.random.choice(arr) < 0.21:  \n",
    "        arm4 = 0.2    #20% chance\n",
    "    elif np.random.choice(arr) < 0.41:\n",
    "        arm4 = 0.5    #20% chance\n",
    "    elif np.random.choice(arr) < 0.61:\n",
    "        arm4 = 0.6    #20% chance\n",
    "    elif np.random.choice(arr) < 0.81:\n",
    "        arm4 = 0.8    #20% chance\n",
    "    elif np.random.choice(arr) < 1.01:  \n",
    "        arm4 = 1    #20% chance\n",
    "    arm4R = np.append(arm4R, arm4)\n",
    "\n",
    "#arm5 dist\n",
    "for i in range(1000):\n",
    "    if np.random.choice(arr) < 0.21:  \n",
    "        arm5 = 0    #20% chance\n",
    "    elif np.random.choice(arr) < 0.41:\n",
    "        arm5 = 0.1    #20% chance\n",
    "    elif np.random.choice(arr) < 0.61:\n",
    "        arm5 = 0.1    #20% chance\n",
    "    elif np.random.choice(arr) < 0.81:\n",
    "        arm5 = 0.3    #20% chance\n",
    "    elif np.random.choice(arr) < 1.01:  \n",
    "        arm5 = 0.4    #20% chance\n",
    "    arm5R = np.append(arm5R, arm5)\n",
    "    \n",
    "    \n",
    "#alt way to get arm5 to be directly dependent on arm 1\n",
    "#arm1 dist\n",
    "# for i in range(1000):\n",
    "#     if np.random.choice(arr) < 0.3:  \n",
    "#         arm1 = 0    #20% chance\n",
    "#     elif sample_prelim.item(2) < 0.5:\n",
    "#         arm1 = 0.1    #20% chance\n",
    "#     elif sample_prelim.item(2) < 0.7:\n",
    "#         arm1 = 0.1    #20% chance\n",
    "#     elif sample_prelim.item(2) < 0.9:\n",
    "#         arm1 = 0.3    #20% chance\n",
    "#     else:  \n",
    "#         arm1 = 0.4    #20% chance\n",
    "#     arm1R = np.append(arm1R, arm1)\n",
    "#     arm5R = np.append(arm5R, arm1) #***this is the crucial line that creates dependency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Round</th>\n",
       "      <th>Arm1 Reward</th>\n",
       "      <th>Arm2 Reward</th>\n",
       "      <th>Arm3 Reward</th>\n",
       "      <th>Arm4 Reward</th>\n",
       "      <th>Arm5 Reward</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Round  Arm1 Reward  Arm2 Reward  Arm3 Reward  Arm4 Reward  Arm5 Reward\n",
       "0      1          0.1          0.4          0.5          0.6          0.3\n",
       "1      2          0.3          0.3          0.2          0.5          0.3\n",
       "2      3          0.3          0.4          0.8          0.8          0.1\n",
       "3      4          0.4          0.4          0.5          0.6          0.0\n",
       "4      5          0.1          0.4          0.6          0.2          0.1\n",
       "5      6          0.1          0.1          1.0          0.5          0.1\n",
       "6      7          0.4          0.4          0.5          0.6          0.1\n",
       "7      8          0.1          0.4          0.6          0.2          0.4\n",
       "8      9          0.1          0.4          0.2          0.6          0.3\n",
       "9     10          0.1          0.3          0.8          0.5          0.0"
      ]
     },
     "execution_count": 259,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2 = {'Round': np.arange(1,1001),\n",
    "        'Arm1 Reward': arm1R,\n",
    "        'Arm2 Reward': arm2R,\n",
    "        'Arm3 Reward': arm3R,\n",
    "        'Arm4 Reward': arm4R, \n",
    "        'Arm5 Reward': arm5R, \n",
    "       }\n",
    "\n",
    "df2 = pd.DataFrame(data2)\n",
    "df2.head(10)  #shows first 10 rounds out of 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "#implementing UCB, 1000 runs\n",
    "#CMAB for K=2 arms being selected\n",
    "rounds = len(df2.index) # number of rounds; \n",
    "k = 5 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "mu_arr = []\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding ucb values. re-initialize to 0 at start of each round \n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt(3/2*math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "        \n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    if r_mu < alpha*opt_mu: #superarm S is bad if the reward sum from S is < alpha*mean Reward of Optimal Arm.\n",
    "        #this superarm is a bad superarm\n",
    "        #only increment the simple arm within this superarm that has min N\n",
    "        if N[arm1_selected] < N[arm2_selected]:\n",
    "            N[arm1_selected] += 1\n",
    "        else:\n",
    "            N[arm2_selected] += 1\n",
    "    #if superarm S is not bad then don't increment N at all:\n",
    "    reward = df2.values[t, arm1_selected+1] + df2.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df2.values[t, arm1_selected+1]  #will need loop here to do this K times\n",
    "    rewardSum[arm2_selected] += df2.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD0CAYAAABgk2Y8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deYBOdf//8eeMfU8JiVvlrk9T3Zbclds0SNZKWlWyDLJkkHIjE79kTUJqKndTIt2losm+M2LQ1BRKl4+UtSLLGIZZzMz1+2NG37lDcc1y5jrX6/HXNedc15zX28y8Op3rXOcEeb1eRETEXYKdDiAiIvlP5S4i4kIqdxERF1K5i4i4kMpdRMSFVO4iIi5U3OkAAAkJCTofU0TEBw0bNgw61/IiUe4ADRs29Ol1Ho+HkJCQfE5TtGnmwKCZA0NeZk5ISDjvOh2WERFxIZW7iIgLqdxFRFxI5S4i4kIqdxERF1K5i4i4kMpdRMSFVO4iIg5Y+dNKTJRhyd4lBfL9Ve4iIoUoPTOdoSuG0mpWK4oFFaPuZXULZDsqdxGRQrLz6E5Cp4fy0oaX6NWwF1/1+oory11ZINsqMpcfEBFxs7nfz6XbvG4UDy7O3A5zeSDkgQLdnvbcRUQKUHJ6MoOWDeKhTx7ihstvYEufLQVe7KA9dxGRApHlzWLm5pk8H/s8+47vo0/DPrzS5hVKFS9VKNtXuYuI5LOk1CQ6ftqRxT8s5tYrb+WDBz/g9r/dXqgZVO4iIvloz7E9tPuwHZ7DHqLaRtH3lr4EBZ3zkusFSuUuIpIPjqcdJ3JVJNFfR1OqWCmWPL6EFte0cCzPBZW7MeY2YIK1tpkx5u/ADMALfAdEWGuzjDE9gd5ABjDGWrvQGFMGeB+oCpwAulprDxXAHCIijln+43L6Le7HT4k/0b1BdyLDIrnqkqsczfSXZ8sYY4YAbwOlcxZNBoZba8OAIKC9MaY6MAAIBVoD440xpYAngW9znvseMDz/RxARccbJ9JP0XtCb1u+3Ji0zjTVd1/BWu7ccL3a4sFMhfwRyn7fTEFib83gJ0AK4FYiz1qZZa5OAnUBd4HZg6R+eKyLi9+L2xlFvWj2iv45mcOPB7Oi3g7DaYU7H+t1flru1di5wOteiIGvtmRtanwAqARWBpFzPOdfyM8tERPxWakYqQ1cMJezdMDK9mazpuoaXWr5UaKc4Xihf3lDNyvW4AnAMOJ7z+M+Wn1l2Th6Px4cokJqa6vNr/ZVmDgyauej5PvF7nv3iWXYe38nD1zzMkHpDKJdSLk+ZC2pmX8r9G2NMM2ttLNAWWAPEA2ONMaWBUkAI2W+2xgF35axvC6w73zf19e7fult6YNDMgaGoznw68zTj149n9Oejubzs5SzuuJi217bNl++dl5kTEhLOu86Xch8ERBtjSgIeYI61NtMY8yrZ5R0MPGetTTXGvAnMNMasB9KBjj5sT0TEMZ5DHrp81oWvfvmKx256jKi7ori0zKVOx/pLF1Tu1trdQKOcxzuApud4TjQQ/Ydlp4CH85xSRKSQZXmzeGXTK0SuiqR8yfJ8/NDHPHyj/9SZPsQkIvIHa3at4clFT2KPWNpd14632r1F9fLVnY51UXRVSBGRHEdTjtJtXjeav9ecjKwMPnzwQ+Y9Os/vih205y4iAmRfbz1icQSHTx1m2O3DiAzLPhzjr1TuIhLQfj3xKxGLI4jZHsPNV9zM0k5LqV+9vtOx8kzlLiIB6/M9n9NxbkeOpBxhQosJPPOvZyge7I5adMcUIiIX4XjacZ5b9RxRX0ZRs2JNNvbY6Iq99dxU7iISULYc2EKHOR3YeXQn/W/tz4stXqRsibJOx8p3KncRCQjpmem8+eWbDF05lEvLXMrqLqtpetVZH9lxDZW7iLjexn0b6TG/B57DHu669i5mtJ/B5eUudzpWgdJ57iLiWokpifRe0JvQ6aEkpycz5+E5LHhsgeuLHbTnLiIuFbc3js4xndl9bDd9b+nL+DvHU6FUhb9+oUuo3EXEVU5nnmb056MZu24stSvVZn339TSu1djpWIVO5S4irhH/czz9l/Qn/ud4utTrwmttX6NiqYpOx3KEyl1E/JrX62XpzqW89fVbfLb9M6qUrcJHD31Ehxs7OB3NUSp3EfFbJ9JO0GthL2Z/N5tKpSoxqtkoBjYaGFDH1s9H5S4ifmnT/k08OudR9ibtZWzzsQz616Aidx9TJ6ncRcSvJKYkMnTlUGZsnkGtSrVY120doX8LdTpWkaNyFxG/seqnVYTPC+dA8gH6NOzDyGYjuazsZU7HKpJU7iJS5KVmpBK5KpIpm6ZgLjNs7LGRf9b4p9OxijSVu4gUaZv2b6LXgl58+9u39LulHxNaTnDlhb7ym8pdRIqknUd3Mmj5IObb+VQrV43FHRfT9tq2TsfyGyp3ESlS0jLSeD72eaZsmkLJYiUZ13wc/W/r79e3vHOCyl1Eiowfkn7g0bcfZevBrYTXD2dc83FcUeEKp2P5JZW7iDguy5vFK5te4dmVz1K5TGUWPraQu6+72+lYfk3lLiKO2pu0l/DPwlmzew3NazRndsfZAXFJ3oKmchcRR3i9Xj749gMiFkeQ6c3knXvfoVGpRir2fKKbdYhIoTty6giPzHmETjGduKnqTWzps4XuDboTFBTkdDTX0J67iBSqpTuX0n1edw6fOsz4O8czuPFgigUXczqW66jcRaRQ7Evax9PLnmauZy43XH4DizouosEVDZyO5VoqdxEpUF6vl1lbZzFgyQBOZ51mZNORDL19KKWLl3Y6mqv5VO7GmBLATOAqIBPoCWQAMwAv8B0QYa3NMsb0BHrnrB9jrV2Y99gi4g9STqfQe2FvZm2dxe1/u50Z7WdQ59I6TscKCL6+oXoXUNxa2xgYBYwFJgPDrbVhQBDQ3hhTHRgAhAKtgfHGGF1wWSQArN61mnrT6vH+1vd5odkLxHaNVbEXIl8Py+wAihtjgoGKwGmgEbA2Z/0SoBXZe/Vx1to0IM0YsxOoC3yZp9QiUmQdOnmIf6/4N+9teY86leuwovMK7rzmTqdjBRxfyz2Z7EMy24EqwD1AE2utN2f9CaAS2cWflOt1Z5afxePx+BQkNTXV59f6K80cGPxt5v3J+5mxYwbz98wnNSOVXiG96BPSh9JppS94Dn+bOT8U1My+lvvTwDJr7TBjTC1gNVAy1/oKwDHgeM7jPy4/S0hIiE9BPB6Pz6/1V5o5MPjLzCfSTjBu3TimbJoCwIM3PMjwsOGEXH7x2f1l5vyUl5kTEhLOu87Xck8k+1AMwFGgBPCNMaaZtTYWaAusAeKBscaY0kApIITsN1tFxAW+2P8FnWI6sfPoTrrU68K45uO4suKVTscSfC/3KcB0Y8w6svfYI4GvgGhjTEnAA8yx1mYaY14F1pH95u1z1trUfMgtIg7afng7I2NH8tG2j6hRoQZrw9fSpHYTp2NJLj6Vu7U2GehwjlVNz/HcaCDal+2ISNGSkZVB5KpIJm2cRJniZRgeNpwhoUOoUKrCX79YCpU+xCQiF+TM9WBW7VpFr5t7Mab5GF3kqwhTuYvIn/J6vXzq+ZR+S/pxNOUo0++dTrcG3ZyOJX9B5S4i5/Xbyd/ou6gvcz1zaVC9AYs7Ltb1YPyEyl1EzpJyOoUpm6YweeNkTqSf4MU7X2RQ40EUD1Zl+Av9pETkd16vl9nfzWboyqHsO76P1nVaM6nVJG6seqPT0eQiqdxFBMg+BNNrQS/m2XncfMXNzLp/Fk2vOusEOPETKncRYb6dzxPzn+B42nEmtZrEwEYDCQ7Sjdr8mcpdJIBtPbiVp5Y+RezuWOpXr8/q+1dzU9WbnI4l+UD/aRYJQBlZGYxeO5qGbzVk22/bmNhyIl888YWK3UW05y4SYOxhS5fPuhD/czyP3vQoUW2juKzsZU7HknymchcJENt+28b7W99n6hdTKV28NLMfnM0jNz3idCwpICp3EZf78eiPDFg6gMU/LCaIIDrc2IHJrSdTo0INp6NJAVK5i7hUyukUJsRN4MX1L1KiWAlevPNFOtXtpEvyBgiVu4gLffXLVzw29zF2Ht3JIzc+wqRWk1TqAUblLuIiXq+Xdze/S99FfalariorOq+gxTUtnI4lDlC5i7jEryd+pfv87izduZSmtZsyt8NcnQUTwFTuIn7O6/Wy7MdldInpQnJ6Mq+1fY2+t/TVJ0wDnMpdxI/Zw5b+S/qz4qcV3Hj5jawNX+vTjanFfVTuIn7o1OlTjP18LBM3TKRsibJMbjWZ3v/sTdkSZZ2OJkWEyl3EzyzasYiIxRHsSdpD57qdmdhyItXKV3M6lhQxKncRP5Gcnswzy54h+utobrj8BmK7xuqSvHJeKncRP7Bp/yY6x3Tmx6M/MjR0KKPuGEXJYiWdjiVFmMpdpAg7dPIQY9eNJSo+iisrXklseCxNajdxOpb4AZW7SBGUnpnOtO+n8U7MO6RkpNCjQQ8mtpxIpdKVnI4mfkLlLlLExP8czxPzn+Db377lgZAHGHPHGJ3eKBdNn3IQKSK8Xi9TNk4hdHooiamJRIVGMbfDXBW7+ER77iJFwMHkgzy56ElitsfQ3rRnxn0z+HXXr07HEj+mchdxkNfrZdbWWQxcOpCTp08yseVEBv1rEEFBQfyKyl18p3IXcUhyejJ9Fvbhv9/+l9Baobx979tcX+V6p2OJS/hc7saYYcC9QEngDWAtMAPwAt8BEdbaLGNMT6A3kAGMsdYuzGtoEX+3dvda+izqgz1sGdVsFJFhkRQLLuZ0LHERn95QNcY0AxoDoUBToBYwGRhurQ0DgoD2xpjqwICc57UGxhtjSuVDbhG/lJGVwai1o2j+XnOSUpNY2WUlI5qOULFLvvN1z7018C0QA1QEBgM9yd57B1gCtAIygThrbRqQZozZCdQFvsxLaBF/FP9zPAOXDmTj/o10/EdH/nPPfyhfsrzTscSlfC33KkBt4B7gamA+EGyt9easPwFUIrv4k3K97szys3g8Hp+CpKam+vxaf6WZ/cvJ0yeJ2hbFzB0zqVCiAhMbTeTuv93Nvh/3/enr/HlmX2nm/ONruR8Btltr0wFrjEkl+9DMGRWAY8DxnMd/XH6WkBDfzuX1eDw+v9ZfaWb/sWjHIvou68vepL30btibCS0mXPCnTP115rzQzBcnISHhvOt8/RDTeqCNMSbIGFMDKAesyjkWD9AWWAfEA2HGmNLGmEpACNlvtoq42oHkA3T4pAP3fHgP5UuWZ123dUy7Z5ouHyCFxqc9d2vtQmNME7LLOxiIAHYB0caYkoAHmGOtzTTGvEp20QcDz1lrU/MnukjRk+XNYmLcRJ6PfR6AMXeMYXDoYF3BUQqdz6dCWmuHnGPxWReXttZGA9G+bkfEXxxIPkDHuR1Zs3sN911/Hy/e+SKminE6lgQofYhJJB9s2LeBjnM7cujUId659x261e9GUFCQ07EkgOnCYSJ5kHI6hcHLBxP2bhhBQUHEdo2le4PuKnZxnPbcRXzg9XqJ/jqa4auHc+jUIZ5o8ASTWk+iYqmKTkcTAVTuIhft1xO/0nNBTxb9sIhmVzVjVLNRhNUOczqWyP9QuYtchNnfzabvor6kZKTwSutX6H9bf4KDdHRTih6Vu8gFOHLqCH0X9+XjbR9z25W3MfO+mToTRoo0lbvIX1i4YyE9F/TkyKkjjG0+liGhQygerD8dKdr0GypyHsfTjvP00qeZvnk6davVZenjS6lXvZ7TsUQuiMpd5BzW7FpD+Lxw9h/fz7Dbh/F80+cpVVxXqxb/oXIXyeXU6VMMWzmMV+Nf5brLriOuexyNajZyOpbIRVO5i+TYtH8TXT/ryo4jOxhw6wDGtxhP2RJlnY4l4hOVuwS8oylHGbJiCO9ufpeaFWuyqssqml/d3OlYInmicpeA9tF3H/HM8mc4dPIQ/W7px+jmo/UpU3EFlbsEpONpx4lcFcnrX75OvWr1mP/ofBrWaOh0LJF8o3KXgLNwx0L6L+nPnmN7GHjbQF5q+RIlipVwOpZIvlK5S8A4mnKU51Y9x7SEadSpXIf13dfTuFZjp2OJFAiVu7heemY6r8e/zujPR3Ms9RhPN3qaF1u8qLsjiaup3MW1vF4vs7bOYmTsSHYd20WrOq2Y2HIidavVdTqaSIFTuYsrHUw+yJOLniRmeww3X3Ezb9z9Bm3+3sbpWCKFRuUurrNx30Ye//RxDiQfYMwdYxgWNkyX5ZWAo3IX10jPTGdk7EgmxE2gVsVarOm6httq3uZ0LBFHqNzFFbYe3EqXmC5sObiFHg16MLn1ZH0YSQKayl38WmZWJpM2TmLEmhFULl2Z+Y/Op51p53QsEcep3MVv7UvaR+eYzqzds5YHQx5k2j3TqFK2itOxRIoElbv4pfl2PuGfhZOemc70e6cTXj+coKAgp2OJFBk6hUD8SpY3i5GxI2k/uz11Lq3D5j6b6dagm4pd5A+05y5+Iyk1ia6fdWWenUd4/XDevPtNShcv7XQskSJJ5S5+4evDX9NmWRt+Pv4zU9tMpf+t/bW3LvInVO5SpHm9Xl6Lf41BawZR+5LaxHWP07nrIhcgT+VujKkKJAAtgQxgBuAFvgMirLVZxpieQO+c9WOstQvzlFgCxs6jO3ly0ZOs/Gkld9S4g5jOMVQqXcnpWCJ+wec3VI0xJYD/ACk5iyYDw621YUAQ0N4YUx0YAIQCrYHxxhjdQl7+VEZWBi9veJl/vPkP4n+OJ6ptFK+FvqZiF7kIeTlb5mVgGvBLztcNgbU5j5cALYBbgThrbZq1NgnYCeiSfHJe9rCl8TuNGbxiMK3qtOL7vt8TcWuErg0jcpF8OixjjAkHDllrlxljhuUsDrLWenMenwAqARWBpFwvPbP8LB6Px5copKam+vxaf+XGmY+kHmHmjpm8t+M9yhQvw6RGk2hTqw3Hfz7O8Z+Pu3Lmv6KZA0NBzezrMffugNcY0wKoD7wHVM21vgJwDDie8/iPy88SEhLiUxCPx+Pza/2Vm2bOzMrk5Q0vM/rz0Zw8fZJOdTsxocUEalSo8T/Pc9PMF0ozB4a8zJyQkHDedT6Vu7W2yZnHxphYoA8w0RjTzFobC7QF1gDxwFhjTGmgFBBC9putIhw6eYiHPnmIz/d8zn3X38f4O8dzfZXrnY4l4gr5eSrkICDaGFMS8ABzrLWZxphXgXVkH99/zlqbmo/bFD/k9Xr51PMp/Zf0JzE1kZn3zaRz3c46b10kH+W53K21zXJ92fQc66OB6LxuR9xhz7E99FvSj4U7FlK/en0WdVxEgysaOB1LxHX0ISYpFBlZGUzdNJX/F/v/AJjUahIDbhtA8WD9CooUBP1lSYFL+CWB7vO7s/XgVu657h6i2kZR+5LaTscScTWVuxSYtIw0Rq0dxUsbXqJauWrM7TCX+6+/X8fWRQqByl0KxBf7v6DH/B5sO7SNrvW6Mrn1ZC4tc6nTsUQChspd8tWJtBNErork9S9fp0aFGizuuJi217Z1OpZIwFG5S75Z/uNyei7oyb6kffS7tR9jm4+lQqkKf/1CEcl3KnfJs8SURJ5Z/gwzNs/g+irXs67bOkL/Fup0LJGApnKXPPnU8ykRiyM4dPIQkbdHMqLpCN0dSaQIULmLT7Ye3MrI2JHEbI+hQfUGLHl8CfWr13c6lojkULnLRTl1+hRTNk5hxJoRlCtZjnHNx/Hvxv+mRLESTkcTkVxU7nLBlv+4nF4LerEnaQ8P3fAQb93zFpXLVHY6loicg8pd/tLRlKMMWj6IGZtnYC4zrO6ymjuuvsPpWCLyJ1Tu8qfmfj+XiMURHD51mOfCnmN4k+F6w1TED6jc5Zx2H9tNv8X9WPTDIm6+4maWdlqqN0xF/IjKXc6yaf8mOnzSgaS0JF5u+TJPNXpKV28U8TP6i5XfpWemM2rtKMavH0/NijWJ7Rqra62L+CmVuwDw/aHv6fRpJ7458A3d6nfjlTavULFURadjiYiPVO4B7tTpU0zdNJXRn4+mfMnyfPbIZ7S/vr3TsUQkj1TuAWzrwa3c/cHd7D++n/amPW/e/SZXVLjC6Vgikg9U7gEoPTOdqPgoIldFUrlMZZZ3Wk7LOi2djiUi+UjlHkC8Xi+rd61m0PJBbDm4hXvNvUS3i6ZquapORxORfKZyDxA/Jf5E93ndWbtnLTUr1uT9+9+n4z866pZ3Ii6lcne505mnmbRxEi+sfYESwSWIahtFj5t76FOmIi6ncnexDfs20Hthb7777TseCHmAqW2mUrNiTadjiUghULm7UGJKIs+ufJa3vn6LWhVrMe/Redxr7nU6logUIpW7i2R5s/jw2w95ZvkzHD51mGcaPcMLd7xA+ZLlnY4mIoVM5e4S8T/H02tBL7Yc3MItNW5h6eNLdekAkQAW7HQAyZssbxYT1k8gdHooR1OOMuv+WWzssVHFLhLgtOfux3458QtdP+vKyp9W6s5IIvI/fCp3Y0wJYDpwFVAKGAN8D8wAvMB3QIS1NssY0xPoDWQAY6y1C/MeO7B5vV4+/PZD+i3pR8rpFKLbRdOjQQ+dsy4iv/P1sEwn4Ii1NgxoC0QBk4HhOcuCgPbGmOrAACAUaA2MN8aUynvswJWYksiADQPo+GlH/n7p3/mm9zc8cfMTKnYR+R++Hpb5BJiT6+sMoCGwNufrJUArIBOIs9amAWnGmJ1AXeBLH7cb0Fb9tIrweeEcSD7Ayy1fZmCjgRQLLuZ0LBEpgnwqd2ttMoAxpgLZJT8ceNla6815ygmgElARSMr10jPLz+LxeHyJQmpqqs+v9RepGalM/nYy7//wPldXuJp3Q9+lYeWG7LA7nI5WaALh5/xHmjkwFNTMPr+haoypBcQAb1hrPzDGvJRrdQXgGHA85/Efl58lJCTEpxwej8fn1/qDL3/+ki6fdWH74e0MuHUA41uMZ8/OPa6e+Vzc/nM+F80cGPIyc0JCwnnX+XTM3RhTDVgODLXWTs9Z/I0xplnO47bAOiAeCDPGlDbGVAJCyH6zVf7C6czTvBD7Av96518kpyezovMKpradStkSZZ2OJiJ+wNc990igMjDCGDMiZ9lTwKvGmJKAB5hjrc00xrxKdtEHA89Za1PzGtrtNh/YzOOfPp5967u6nXit7WtcUvoSp2OJiB/x9Zj7U2SX+R81Pcdzo4FoX7YTaE6mn2RC3ATGrRtHlbJVdE0YEfGZPsRURMTujqXrZ13Zm7SXx//xOFPbTOWyspc5HUtE/JTK3WHbftvGwGUDWfnTSq6pfA2fh39OWO0wp2OJiJ9TuTskMyuTiRsm8sLa7Ks2TmgxgYhbIihXspzT0UTEBVTuDth9bDedYzqzfu96Hgh5gDfueoNq5as5HUtEXETlXoi8Xi/vbXmPAUsHZD++7z061e2kSweISL5TuReSLQe2ELE4grh9cYTWCmXW/bO4uvLVTscSEZdSuRew1IxURq8dzYS4CVQuU5nodtF0b9Cd4CBdSl9ECo7KvQCt37ueJ+Y/gT1iCa8fzqRWk7i0zKVOxxKRAKByLwC7j+1mYtxE3vjqDWpXqs2yTstoVaeV07FEJICo3PPRweSDjF8/nje+fIOMrAwG3DqAsXeO1Q2qRaTQqdzzwZFTR5i4YSKvxb9GWkYa4fXDGdFkBLUvqe10NBEJUCr3PNqwbwMPfPQAv538jcf+8RjPN32e6y67zulYIhLgVO4+8nq9RH8dTb/F/ah9SW2WdlpK/er1nY4lIgKo3H2SmJJIn0V9+Hjbx7Sq04rZD86mcpnKTscSEfmdyv0irfxpJT3m9+CXE78wrvk4hoQO0X1MRaTIUblfoN3HdtN7YW+W/7icay+9lg3dN3DLlbc4HUtE5JxU7hdg3vZ5hM8Lx+v18nLLl+l7S1/KlCjjdCwRkfNSuf+J1IxUIldFMmXTFBpe0ZCPH/6Yaypf43QsEZG/pHI/B6/Xy4IdC4hcFcm2Q9uIuCWCSa0mUap4KaejiYhcEJX7H+xK3EWvhb1Y+dNKalasyZLHl9Dm722cjiUiclFU7jkyszKJio8icnUkwUHBvH7X6/Rq2IviwfonEhH/o+Yi+0yYhz95mK9++Yq7rr2LaXdPo1alWk7HEhHxWcCX+3+3/pfeC3tTLLgYHz30EQ/f8LDujCQifi9gyz01I5Wnlz7NtIRphP0tjJn3zdSdkUTENQKy3BftWMSQlUP4/tD3DGk8hLF3jtWxdRFxlYBqtK0HtzJs1TAW/7CYay+9lvmPzqedaed0LBGRfBcQ5Z6UmsTgFYN5++u3qViqIhNbTmTAbQMoWayk09FERAqEq8vd6/USsz2GAUsG8Gvyrzzd6GmGNxmuKziKiOu5tty/2P8Fg5YPIm5fHDdVvYmYR2J0oS8RCRgFXu7GmGDgDaAekAY8Ya3dWVDb25W4i8jVkcz+bjbVylXjrXveoluDbnrDVEQCSmE03n1AaWvtv4wxjYBJQPv83si237bx9tdv88ZXb1AsqBgjmoxgcOPBVChVIb83JSJS5BVGud8OLAWw1m4yxvwzv75xljeLj3/8mOUbl7Nh3waCCCK8fjij7xjNlRWvzK/NiIj4ncIo94pAUq6vM40xxa21GXn9xpsPbGZkwkhuuPwGXmrxEp3rdaZ6+ep5/bYiIn6vMMr9OJD72EjwuYrd4/Fc9DcuQxli28RyeYXLCQoKInFfIokk5iGqf0hNTfXp38ufaebAoJnzT2GUexzQDvg455j7t+d6UkhIiG/f3ZOH1/opj8ejmQOAZg4MeZk5ISHhvOsKo9xjgJbGmA1AENCtELYpIhLQCrzcrbVZQJ+C3o6IiPyfYKcDiIhI/lO5i4i4kMpdRMSFVO4iIi6kchcRcaEgr9frdAYSEhKcDyEi4ocaNmx4zps+F4lyFxGR/KXDMiIiLqRyFxFxIb+9g0Vh3wSkMBljSgDTgauAUsAY4HtgBuAFvgMirLVZxpieQG8gAxhjrV3oROb8YoypCiQALcmeaQYuntkYMwy4FyhJ9u/zWlw8c87v9kyyf7czgZ64+OdsjLkNmGCtbWaM+TsXOKcxpgzwPlAVOAF0tdYeupht+/Oe++83AQGeJfsmIG7RCThirQ0D2gJRwGRgeM6yIMFEUKIAAAJ7SURBVKC9MaY6MAAIBVoD440xpRzKnGc5f/j/AVJyFrl6ZmNMM6Ax2bM0BWrh8pmBu4Di1trGwChgLC6d2RgzBHgbKJ2z6GLmfBL4Nue57wHDL3b7/lzu/3MTECDfbgJSBHwCjMj1dQbQkOy9OoAlQAvgViDOWptmrU0CdgJ1CzNoPnsZmAb8kvO122duTfZVUmOABcBC3D/zDqB4zv95VwRO496ZfwQeyPX1xcz5e7/leu5F8edyP+dNQJwKk5+stcnW2hPGmArAHLL/qx1krT1zatMJoBJn/xucWe53jDHhwCFr7bJci109M1CF7J2Sh8m+uN5/yb7fgZtnTib7kMx2IBp4FZf+nK21c8n+j9cZFzNn7uU+ze7P5X5BNwHxV8aYWsAaYJa19gMgK9fqCsAxzv43OLPcH3Un+9LQsUB9sv9XtGqu9W6c+QiwzFqbbq21QCr/+0fsxpmfJnvm68h+v2wm2e83nOHGmc+4mL/h3Mt9mt2fyz2O7ON3/NlNQPyRMaYasBwYaq2dnrP4m5xjtJB9HH4dEA+EGWNKG2MqASFkv1Hjd6y1Tay1Ta21zYDNQBdgiZtnBtYDbYwxQcaYGkA5YJXLZ07k//ZIjwIlcPnvdi4XM+fv/ZbruRfFnw9juPkmIJFAZWCEMebMsfengFeNMSUBDzDHWptpjHmV7B98MPCctTbVkcQFYxAQ7daZc86KaEL2H3gwEAHswsUzA1OA6caYdWTvsUcCX+Humc+44N9nY8ybwExjzHogHeh4sRvTJ1RFRFzInw/LiIjIeajcRURcSOUuIuJCKncRERdSuYuIuJDKXUTEhVTuIiIupHIXEXGh/w9BnLyq2iY+5AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "reward_arr1 = UCB_cumu_reward\n",
    "\n",
    "plt.plot(reward_arr1,color = 'g')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Number of times Each Arm was Selected')"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXEAAAECCAYAAAAIMefLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWRElEQVR4nO3df5xddX3n8dckIQlqEnFdhfpQaYt+Oi0CEhQQIynLjwVBFKpUS138gVipaEUFAV3dxQUVaKGtikEbbaW2RbAIRbBYEBBFR1B59PpxMeWxPpb6iy0BlAkkmf3j+x25TCaZycy9M/NlXs/HYx4595x7zvl8zz3nfc/9nnNvBkZGRpAktWnBbBcgSZo6Q1ySGmaIS1LDDHFJapghLkkNM8QlqWGLZruAx6uI2BX4N+CNmfnJrvHvBHbPzBN6tJ67gd/LzG/1YnkTrGs5cA3wZOC9mXl517SXAvtm5vsi4mXAwZl5Sr9r6lr/CcCFlG3e7buZ+dopLnME+M+Z+fNJPPco4Erg9zPz76ayvtkWEe8GXgMMAAuBLwFnZObDE8x3N1PcByPi14HzMvPY7Zyvp8dRywzx/toMnB8RN2dmznYxPbAX8PTM3G2caS8AngKQmVdSAm2m3ZSZR87CegHeAnwW+BOguRCPiFcCrwD2z8yHImIpcBnwfuCMPq762UD0cfmPe4Z4fz0EnA9cGhH7jz2jiYi1wJ2Zed7Yx/Xs5lLgIGAn4MPAAcBK4BHgZZl5T13UyRGxJ7AEOD8zP1WXdxRwFrAY+CXwzsy8NSLeD+wP/Brwncw8fkxdLwf+O6W77QHgHcB64FPAMyLiDurBXp+/L/BmYGFErAf+N+XM7MiIuAEYAvYDngZ8AtgZOBB4IvCqzPxeRKygnEk/D9gBuB54V2ZujIgPUALmYeBe4ITM/PfteB2IiOcCfwksA3YB7gCOy8zhWv9FtZ6H63b6Sp31AxGxH/CfgI9k5l+Os+zfAFZTAqkTEftl5tfrtLWUN7ffBK4Cnk55LZ5Xh6+sbTqqbpc3dq17dPl3AKdm5vUR8Wrgr4CdatheUrfv9dto32S23y6Us+8dgYfqfH9Mec2IiMXAhyiv20LgduCUzLx/TK1b2+cWUfbhI4GNwNeAk4FLKPvUtZl5WES8qK7nicAm4AOZeVVE7FBfo0OAnwI/oeyT85594v33QeBB4H9NYd6lmbkf8D5K+F2YmXsCPwJO6HreQ5m5N2UHPycificinlPXeURmPh94E3B5RDyxzvNs4PnjBPhvAR8Hjq3reh/wj8C/A28EfpiZe40GOEBmfqPO83eZeeY47dg1Mw8AjqccyDdk5j6Uj+tvrc/5U2AoM1cCzweeCrwjIp4JvB14QZ3nOmDfrWyvVRFxx5i/19VpJwKfrttzN+DXgZfWcPgC8D8yc/f6vAsjYvTYWFdregXlU9UO46z3zcDVmflT4HOUs/FuT8jM38nM0+rjvSlvzi8BTgUezMwXUd7ETh9n+ZcDh9fh/wr8R23rAHAEcMU22jfZ7fdp4D7gxxFxa0ScDzwrM2+r00+nhO/Kul/cA5zbvYAJ9rm3UE5A9gR2p7zZvIpH96nDImInyhvUH9b9+WjgYxHxrDr/c4HfpuznzxqnDfOSZ+J9lpmbI+J44I6IuHY7Z/98/feHwI8z8ztdj5/S9byL67ruiYjrgP9COeB2Aa6P+NWn1c2UAxzg65m5cZx1HgRcn5nr6jK/EhE/pRyAU/2NhtG+8x/Wf7/U9Xh1HT4SeGFEvKE+3rH+ex7wHeDbEXENcE1mXr+V9WyrO+U04JDa7/tcyqeQJ1HOiDdl5tUAmTlUx1G326V1/json3SWU85mqc9ZArwOeH0d9Wngloh4Zmb+qI67eUwtX8zMRyiB+Ysx2+MpbOkK4HMR8S5gFXABJcgeoATgjyNia+37v0xi+2XmeuDQ+qnidymvy9UR8dH65nMk5VrIIXW7LKacEXc7hK3vcwcDf9315n9c3X6ru+bfv87/ha75R4A96vyX1k+zD0fEZ+v4ec8QnwGZ+aOIOIlygH+ma9II5SLSqMVjZt3QNfzINlaxqWt4QX3uIkoYHzc6oZ6V3UM5q3xwK8tayJZhvYDSxbHNC1zb0N0OaoCNt95XZman1vpkYKS+CR4I7EM5kP80Ir6Ume/ezhr+lrJN/h64mnImN0B5s3tMeyNid+D79eEjteaRGizdrxeUs8mdgL+IiD+v40YonzBGaxy7rTeMebyt15ba3bQYeBmlq+qLlH73jZR+6622b7Lbr4b/zZn5NWAd8MmIeDHlDeY0yuvztsy8pj7/ScDSMaUuZOv73GO2c0Q8nS17AhYCnczct+t5vwb8DDiJx2778U5A5iW7U2ZIZl5GubPj7V2jf0Y5uEZ31gOnuPgT6jKeRTlQr69/h9buESLiCOC7PHqGuzXXA4fVMzIi4iDgmcA3JphvIyXop+pa4E8iYqCe3V4J/HHt67+TcnCfQ+l2ecEUln8Ypctk9KLjvpTQSGAkIg4BiIi9ga8w+WPjj4APZuazM3PXzNyV0r1yYlfXVS9cQem+uC4zvw+sAP6ARz/ljNu+7dh+TwDOjYjuTwLPA75dh6+lvB6La1fTGuCcMcvY1j73z8BrImJJnf9jwKt57H7zdeA5EfGSOv9elDetZ1COnddGxNJ60fU4BHgmPtNOAV7c9fjPgc9GRAJ3U8JjKpZGxLcpZ/JvzcwfAETEmygfw0fPOF+WmQ92fVTdQmb+a0S8hdKXuYhyceqozFy/rflq7ZfWs9GhKbThFEqf8PcoB/U/Ax/OzEci4u+Bb0XEg5SLxVu7dXFVvQjYbWPtCz4DuKJ2X6wHbgR2y8wNEXEM8GcR8RHKp41jMvPhCdpLDci9KGfI3T5Dubh3wiTaPVlXAO8CvlwffxnYo6vLZmvt++Qkt9//pHR9fC3KrZULgW9SPmmMTj+PckFzIaV76dTuBdR9Z2v73MXArpR9YwC4gXKhcjkwHBG3Ud54jgU+UoN6AaV//O46/26UN6R7KeEuYMCfopWkdtmdIkkNM8QlqWGGuCQ1zBCXpIZN6u6UiLidR7/i+m+UbyGupdz3eSdwcr0f9UTK/ZwbgbMz86qeVyxJ+pUJ706pt/rcWr9GOzruSuCCzLwhIj5OuYf0VsptT/tQvgRwM7BPZj7miw1DQ0PeDiNJ22nlypVjv2gGTO5MfE/gCfXr3Iso96OupNyHCuUm/EMp3xq8pYb2hoi4i/K12G+OU8x2N2CmdDodBgcHZ7uMWTOf22/b52fbYe63f2ho61+9mEyI/5Jyk/8lwHMooT2QmaNn1A9Qvj22nMf+qtjo+C10Op1JrHZ2DA8Pz+n6+m0+t9+2z8+2Q9vtn0yI/wC4q4b2DyLiXsqZ+KhllF8/u78Ojx2/hbn8jjfX35H7bT6337bPz7bD3G//ts7EJ3N3yuspv4k9+vsey4Hrun597HDgJuA2yteel0b5behBykVPSVKfTOZM/JPA2oi4mXI3yuuBnwNr6i+rdYDLMnNTRFxECfQFwJmZOdynuiVJTCLE6+/3vmacSVv84l5mrqH8upkkaQb4ZR9JapghLkkNM8QlqWH+pxBqwq6nXz1Da1rX16Xffe5L+7p8zT+GuDTHPV7ewMA3sX6wO0WSGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1bNFknhQRTwOGgEOAjcBaYAS4Ezg5MzdHxInASXX62Zl5VV8qliT9yoRn4hGxA3Ax8FAddQFwVmauAgaAoyNiZ+AU4ADgMOCciFjSn5IlSaMm051yHvBx4J76eCVwYx2+BjgYeCFwS2ZuyMz1wF3AHj2uVZI0xja7UyLiBOBnmXltRLynjh7IzJE6/ACwAlgOrO+adXT8uDqdzpQL7rfh4eE5XV+/zff299t837Zztf0t7/cT9Ym/HhiJiIOBvYDPAE/rmr4MuA+4vw6PHT+uwcHBKRU7Ezqdzpyur9/mbvvXzXYBPTG1bfv4aDvM3WN/7u73xdDQ0FanbTPEM/Mlo8MRcQPwZuAjEbE6M28ADgf+BbgN+GBELAWWAIOUi56SpD6a1N0pY5wKrImIxUAHuCwzN0XERcBNlH72MzNzuId1SpLGMekQz8zVXQ8PHGf6GmBND2qSJE2SX/aRpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIatmiiJ0TEQmANEMAm4HXAALAWGAHuBE7OzM0RcSJwErARODszr+pT3ZIkJncmfhRAZh4AvA+4oP6dlZmrKIF+dETsDJwCHAAcBpwTEUv6UrUkCZhEiGfmF4A31YfPBn4CrARurOOuAQ4GXgjckpkbMnM9cBewR88rliT9yoTdKQCZuTEiPg28Avg94MjMHKmTHwBWAMuB9V2zjY7fQqfTmXLB/TY8PDyn6+u3+d7+fpvv23autr/l/X5SIQ6Qmf8tIk4DvgHs2DVpGXAfcH8dHjt+C4ODg9tf6QzpdDpzur5+m7vtXzfbBfTE1Lbt46PtMHeP/bm73xdDQ0NbnTZhd0pE/GFEvKc+/CWwGfhWRKyu4w4HbgJuA1ZFxNKIWAEMUi56SpL6ZDJn4pcDfxURXwV2AN4OdIA1EbG4Dl+WmZsi4iJKoC8AzszM4T7VLUliEiGemb8AXjXOpAPHee4ayu2IkqQZ4Jd9JKlhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhi3a1sSI2AH4FLArsAQ4G/hXYC0wAtwJnJyZmyPiROAkYCNwdmZe1b+yJUkw8Zn48cC9mbkKOBz4C+AC4Kw6bgA4OiJ2Bk4BDgAOA86JiCX9K1uSBBOciQP/AFzW9XgjsBK4sT6+BjgU2ATckpkbgA0RcRewB/DN3pYrSeq2zRDPzAcBImIZJczPAs7LzJH6lAeAFcByYH3XrKPjx9XpdKZRcn8NDw/P6fr6bb63v9/m+7adq+1veb+f6EyciHgmcAXw0cy8NCI+3DV5GXAfcH8dHjt+XIODg1OrdgZ0Op05XV+/zd32r5vtAnpiatv28dF2mLvH/tzd74uhoaGtTttmn3hEPB24DjgtMz9VR98eEavr8OHATcBtwKqIWBoRK4BBykVPSVIfTXQmfgawE/DeiHhvHfc24KKIWAx0gMsyc1NEXEQJ9AXAmZk53K+iJUnFRH3ib6OE9lgHjvPcNcCaHtUlSZoEv+wjSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1bNFknhQR+wIfyszVEbEbsBYYAe4ETs7MzRFxInASsBE4OzOv6lPNkqRqwjPxiHg3cAmwtI66ADgrM1cBA8DREbEzcApwAHAYcE5ELOlPyZKkUZPpTvkhcEzX45XAjXX4GuBg4IXALZm5ITPXA3cBe/SyUEnSlibsTsnMz0fErl2jBjJzpA4/AKwAlgPru54zOn5cnU5n+yudIcPDw3O6vn6b7+3vt/m+bedq+1ve7yfVJz7G5q7hZcB9wP11eOz4cQ0ODk5htTOj0+nM6fr6be62f91sF9ATU9u2j4+2w9w99ufufl8MDQ1tddpU7k65PSJW1+HDgZuA24BVEbE0IlYAg5SLnpKkPprKmfipwJqIWAx0gMsyc1NEXEQJ9AXAmZk53MM6JUnjmFSIZ+bdwH51+AfAgeM8Zw2wppfFSZK2zS/7SFLDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhpmiEtSwwxxSWqYIS5JDTPEJalhhrgkNcwQl6SGGeKS1DBDXJIaZohLUsMMcUlqmCEuSQ0zxCWpYYa4JDXMEJekhhniktQwQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIYZ4pLUMENckhq2aLYL0OTsevrVM7i2dX1d+t3nvrSvy5fmk6ZCfOaCrL8hBgaZpN5oKsQlzS9+Ap1YT0M8IhYAHwX2BDYAb8zMu3q5DknSo3p9YfPlwNLM3B84HTi/x8uXJHXpdYi/GPgSQGZ+Hdinx8uXJHUZGBkZ6dnCIuIS4POZeU19/H+A38jMjaPPGRoa6t0KJWmeWLly5cB443t9YfN+YFnX4wXdAb6tQiRJ26/X3Sm3AEcARMR+wPd6vHxJUpden4lfARwSEV8DBoDX9Xj5kqQuPe0Tf7yLiCcAXwbekJnfn+16ZkpEvBp4O7AJ+C7wlszcPLtVzZyIOJZyt9UI8InMvGSWS5pxEfEJ4P9l5umzXctMioh3AG8AflZHnZSZOYslbcHfTpmkiNgH+Crwm7Ndy0yKiB2Bs4HfzcwXASuAI2e3qpkTEQuBc4GDgf2Bd0XEU2e3qpkVEScBz5vtOmbJ3sBrM3N1/ZtTAQ7z5BubEbEcuAR4MvBUYA1wHOXddSfgbyl9+TsCuwAXAkcDuwPvzMx/BJYArwD+eqbrn47pth34IvCizPxlXeQiYHgGmzAtvXjtI2IwMzdGxNMo3YQPznhDpqBHbd8f2A+4GPitmW7DdPTouF8JvCcidgauzsxzZrodE5kvZ+K7AZ/LzEMpZ5HvqOMvzcyDKd0EyzLzCOBDwB8BxwBvovbrZ+YtmfmjGa98+qbV9szcnJk/AYiItwJPonQptaIXr/3GiDgG+A7l09gjM9uEKZtW2yNiF+D9wMkzXXiPTPu1Bz4HvBk4CHhxRMy5T6HzJcR/DLw8Iv4GOAvYoY7v/mh0e/33PqCTmSPAfwBLZ6zK/ph22yNiQUScBxwCHFunt6Inr31mXg48A1gMvLbfRffIdNv+SsoZ7D9Rrgm8JiJOmIG6e2Va7Y+IAeDPMvPnmfkwcDXw/JkpffLmS4i/E7g1M48H/oHykRig++JcS8G0PXrR9ospB/XLu7pVWjGt9kfE8oi4MSKW1Iu5vxgz71w2rbZn5kWZuTIzV1OuC1yamWv7VGs/THffXw7cGRFPqoF+EDDUl0qnYV70iVP6dT8WEX8A3AtspPRxzwfTantE7E25On8T8JWIALgwM6/oQ639MK32Z+b9EfFZ4KsR8Qjl7py/6UulvTef93uY/mu/PiLOAP6F8oN+12fmP/Wl0mnwFkNJath86U6RpMclQ1ySGmaIS1LDDHFJapghLkkNM8QlqWGGuCQ1zBCXpIb9f/gvsW+B9/pnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(['arm1','arm2','arm3','arm4','arm5'], N) #NB: N is calculated according to CUCB algorithm (page 5 in Superarm paper)\n",
    "plt.title(\"Number of times Each Arm was Selected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [
    {
     "ename": "IndentationError",
     "evalue": "expected an indented block (<ipython-input-212-dbe3899c7e2f>, line 29)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-212-dbe3899c7e2f>\"\u001b[1;36m, line \u001b[1;32m29\u001b[0m\n\u001b[1;33m    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mIndentationError\u001b[0m\u001b[1;31m:\u001b[0m expected an indented block\n"
     ]
    }
   ],
   "source": [
    "#implementing CUCB, 1000 runs\n",
    "#CMAB for any number K arms selected\n",
    "\n",
    "rounds = len(df.index) # number of rounds; \n",
    "k = 10 # number of arms\n",
    "mu = 0 # initialize to 0, the empirical mean\n",
    "N = np.zeros(k) # number of times a given arm has been selected before current round; initialize to 0 as no arm selected yet\n",
    "rewardSum = np.zeros(k)\n",
    "hist_UCB_rewards = [] #stores history of UCB CHOSEN rewards\n",
    "\n",
    "for t in range(rounds):\n",
    "    UCB_Values = np.zeros(k) #array holding ucb values. re-initialize to 0 at start of each round \n",
    "    temp_UCB_Values = np.zeros(k)\n",
    "    arm_selected = 0\n",
    "    for a in range(k):\n",
    "        if (N[a] > 0):\n",
    "            #calculate UCB below\n",
    "            mu = rewardSum[a]/N[a]\n",
    "            ucb_value = mu + math.sqrt(3/2*math.log(t)/N[a]) \n",
    "            UCB_Values[a] = ucb_value\n",
    "        elif (N[a] == 0): #i.e. if N is 0 for arm a, then allow exploration of that arm\n",
    "            UCB_Values[a] = 1e500 #infinity\n",
    "        temp_UCB_Values[a] = UCB_Values[a]\n",
    "    \n",
    "    for i in range(K): #K is the num of arms we are selecting (window size)\n",
    "        for a in range(k): #sort, and take the first K indices\n",
    "            #take the higher K arms in terms of UCB values\n",
    "            #need array with indices of the K best arms\n",
    "    arm1_selected = np.argmax(UCB_Values) #NB: argmax gives index of max value in a list/array\n",
    "    temp_UCB_Values[arm1_selected] = 0\n",
    "    arm2_selected = np.argmax(temp_UCB_Values)\n",
    "    \n",
    "    #update Values as of round t\n",
    "    N[arm1_selected] += 1 #will need loop here to do this K times\n",
    "    N[arm2_selected] += 1\n",
    "    reward = df.values[t, arm1_selected+1] + df.values[t, arm2_selected+1]\n",
    "    rewardSum[arm1_selected] += df.values[t, arm1_selected+1]  #will need loop here to do this K times\n",
    "    rewardSum[arm2_selected] += df.values[t, arm2_selected+1]\n",
    "    hist_UCB_rewards.append(reward)\n",
    "    UCB_cumu_reward = np.cumsum(hist_UCB_rewards)\n",
    "    UCB_total_reward = sum(hist_UCB_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}