VI.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 133,
   "id": "suspected-minute",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Assignment 2 \n",
    "## Gargi Vaidya\n",
    "## Part 4. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "surgical-enough",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convergence Plot of Value Iteration - \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgQklEQVR4nO3de3hc9X3n8fd3ZnTXWLbRxUbyDSyTGgjBKOZihzTdDWvoLk7aboubQoKhjp9C2m633dKnfdLuk6dPm2wvW1oa16SEJE0CaRoab2IuhYTwBAew7DgGA7KF8UXYluWbLFnW/bt/zJE9iJE1si5ndObzep555lx+R/PV8fhzjs75nXPM3RERkeiKhV2AiIhMLgW9iEjEKehFRCJOQS8iEnEKehGRiEuEXUAmlZWVvnDhwrDLEBGZNrZt23bM3asyzcvJoF+4cCGNjY1hlyEiMm2Y2f6R5unQjYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRF5mg7xsY5KEfNvPC7rawSxERySmRCfpEzNj4wl6e2nUk7FJERHJKZILezFhSU86e1o6wSxERySmRCXqA+poku1s70VOzRETOi1bQV5fTfraPts6esEsREckZkQr6JTVJAPa0doZciYhI7ohU0NdXlwPoOL2ISJpIBX1VsoiKkgJ2H9UevYjIkEgFvZlRX11Osw7diIicE6mgh6DnzdEO9bwREQlEL+iryznV1cexzt6wSxERyQmRC/rzPW90QlZEBCIY9PU1Qc8bnZAVEQEiGPTVySJmFCfYrT16EREggkGfuudNUnv0IiKByAU9pA7f7GlVzxsREYhq0FcnOdnVx/Ez6nkjIhLNoA9OyOo4vYhIlkFvZqvMrMnMms3sgQzzV5vZTjPbYWaNZrYy22Ung25uJiJy3qhBb2Zx4CHgVmApsMbMlg5r9hxwjbt/AFgLfGkMy0646mQRyeIEe45qj15EJJs9+uVAs7vvdfde4DFgdXoDd+/082c+ywDPdtnJMNTzZrf26EVEsgr6WuBg2nhLMO1dzOzjZvYm8H1Se/VZLxssvy447NPY1jb+B3zXV5fTrC6WIiJZBb1lmPaefovu/oS7vw/4GPC5sSwbLL/R3RvcvaGqqiqLsi6svibJiTO9HNPTpkQkz2UT9C3AvLTxOuDQSI3d/QXgcjOrHOuyE+n8Q0i0Vy8i+S2boN8K1JvZIjMrBO4ANqU3MLPFZmbB8DKgEDiezbKT5VzPG52QFZE8lxitgbv3m9n9wNNAHHjE3XeZ2fpg/gbgl4G7zKwPOAv8WnByNuOyk/S7vEvNjKDnjfboRSTPjRr0AO6+Gdg8bNqGtOHPA5/PdtmpMPS0KV00JSL5LpJXxg5ZUpNUzxsRyXuRDvrF1eUcP9PLcfW8EZE8FumgP39CVnv1IpK/Ih305542peP0IpLHIh30c2YUkyxK6FYIIpLXIh30ZsbimnL1pReRvBbpoAdYUp1UX3oRyWuRD/r6GvW8EZH8lgdBr543IpLfIh/0S4Z63ijoRSRPRT7oh3reqIuliOSryAf9uZ43OiErInkq8kEPqXvTq4uliOSrvAj6JTVJjnX2cuJMb9iliIhMubwI+sXVuhWCiOSvvAh63dxMRPJZXgT93IpiytXzRkTyVF4EvZmxuLpcNzcTkbyUF0EPqQundOhGRPJR3gR9fXWSY509nFTPGxHJM/kT9LoVgojkqayC3sxWmVmTmTWb2QMZ5n/CzHYGry1mdk3avH1m9qqZ7TCzxoksfiyGbm62WydkRSTPJEZrYGZx4CHgo0ALsNXMNrn762nN3gY+7O4nzexWYCNwfdr8j7j7sQmse8wurSimrDBOs/boRSTPZLNHvxxodve97t4LPAasTm/g7lvc/WQw+hJQN7Fljl/qnjdJ7dGLSN7JJuhrgYNp4y3BtJHcAzyZNu7AM2a2zczWjbSQma0zs0Yza2xra8uirLFbUq2eNyKSf7IJesswzTM2NPsIqaD/w7TJK9x9GXArcJ+Z3ZxpWXff6O4N7t5QVVWVRVljV19TTltHD6e61PNGRPJHNkHfAsxLG68DDg1vZGbvB74ErHb340PT3f1Q8H4UeILUoaBQ6GlTIpKPsgn6rUC9mS0ys0LgDmBTegMzmw98B7jT3XenTS8zs+TQMHAL8NpEFT9W9cHNzXScXkTyyai9bty938zuB54G4sAj7r7LzNYH8zcAnwUuAf7RzAD63b0BqAGeCKYlgG+4+1OT8ptkoXZmCWWFcT2ERETyyqhBD+Dum4HNw6ZtSBu+F7g3w3J7gWuGTw/LUM8bPYRERPJJ3lwZO6S+Wo8VFJH8kndBv6SmnKMdPbR39YVdiojIlMi7oK+vDm6FoMM3IpIn8i/oh25upsM3IpIn8i7oL60oobQwri6WIpI38i7oYzGjvrpcNzcTkbyRd0EPsLhaNzcTkfyRl0Gvnjcikk/yMujPP21Ke/UiEn35GfTVurmZiOSPvAz62pnqeSMi+SMvgz4WMxar542I5Im8DHpIHb7RHr2I5IP8DfqaclpP99B+Vj1vRCTa8jbolwQ9b5rV80ZEIi5vg/7czc10zxsRibi8DframSWUFOhpUyISfXkb9EM9b3TRlIhEXd4GPaROyGqPXkSiLr+DvjrJkdPd6nkjIpGW10F/vueN9upFJLqyCnozW2VmTWbWbGYPZJj/CTPbGby2mNk12S4bpiU1wT1vdOGUiETYqEFvZnHgIeBWYCmwxsyWDmv2NvBhd38/8Dlg4xiWDc25njfaoxeRCMtmj3450Ozue929F3gMWJ3ewN23uPvJYPQloC7bZcM01PNGt0IQkSjLJuhrgYNp4y3BtJHcAzx5kctOOT1WUESiLpugtwzTPGNDs4+QCvo/vIhl15lZo5k1trW1ZVHWxKivSXK4vZvT3ep5IyLRlE3QtwDz0sbrgEPDG5nZ+4EvAavd/fhYlgVw943u3uDuDVVVVdnUPiHqq9XzRkSiLZug3wrUm9kiMysE7gA2pTcws/nAd4A73X33WJYNm3reiEjUJUZr4O79ZnY/8DQQBx5x911mtj6YvwH4LHAJ8I9mBtAf7J1nXHaSfpeLUjerhOKCmK6QFZHIGjXoAdx9M7B52LQNacP3Avdmu2wuOdfzRoduRCSi8vrK2CH11UkduhGRyFLQk7q52eH2bjrU80ZEIkhBDywJHkKiK2RFJIoU9KT26AGadUJWRCJIQQ/UzSqluCCmWyGISCQp6IF4zLi8qlyHbkQkkhT0gSU16nkjItGkoA8sri7nkHreiEgEKegDQ7dC0D1vRCRqFPSBoZub6Ti9iESNgj4wb3aq582Og6fCLkVEZEIp6APxmHHbVXP57k/f0b3pRSRSFPRp7l6xiDO9A/xrY0vYpYiITBgFfZqr6yr44MJZPLrlbQYGMz4IS0Rk2lHQD7N2xSIOnjjLs2+0hl2KiMiEUNAP89GlNdTOLOGRH78ddikiIhNCQT9MIh7jUzct5OW3T7DrUHvY5YiIjJuCPoNf/eA8SgvjfPnFfWGXIiIybgr6DCpKCviV6+rYtOMQbR09YZcjIjIuCvoRfOqmhfQODPL1l/eHXYqIyLgo6EdwWVU5v/C+av7lpf309A+EXY6IyEXLKujNbJWZNZlZs5k9kGH++8zsJ2bWY2a/P2zePjN71cx2mFnjRBU+Fe5esZBjnb1872eHwy5FROSijRr0ZhYHHgJuBZYCa8xs6bBmJ4DfBv5qhB/zEXf/gLs3jKfYqbZycSX11eU88uLbuOsCKhGZnrLZo18ONLv7XnfvBR4DVqc3cPej7r4ViNRNYsyMtSsXsevQaV55+0TY5YiIXJRsgr4WOJg23hJMy5YDz5jZNjNbN5bicsHHr61lVmkBj7yoC6hEZHrKJugtw7SxHMdY4e7LSB36uc/Mbs74IWbrzKzRzBrb2trG8OMnV3FBnDXL5/Mfr7dy8ERX2OWIiIxZNkHfAsxLG68DDmX7Ae5+KHg/CjxB6lBQpnYb3b3B3Ruqqqqy/fFT4s4bFxAz4ytb9oVdiojImGUT9FuBejNbZGaFwB3Apmx+uJmVmVlyaBi4BXjtYosNy9yKEm67ei6Pbz1IZ09/2OWIiIzJqEHv7v3A/cDTwBvAt9x9l5mtN7P1AGY2x8xagN8D/sTMWsxsBlAD/NjMfga8Anzf3Z+arF9mMq1duYiOnn6+3Xhw9MYiIjkkkU0jd98MbB42bUPa8BFSh3SGOw1cM54Cc8UH5s3k2vkzeXTLPu66cSGxWKZTFyIiuUdXxo7B2hWL2He8ix82HQ27FBGRrCnox2DVVXOYW1GsrpYiMq0o6MegIB7jrhsX8mLzcd48cjrsckREsqKgH6M1y+dRXBDjyz/eF3YpIiJZUdCP0czSQn5pWR1P7HiH4526V72I5D4F/UW4+6aF9PYP8s1XDoRdiojIqBT0F6G+JsnNS6r46k/209s/GHY5IiIXpKC/SGtXLORoRw+bX9W96kUktynoL9LN9VVcVlWme9WLSM5T0F+kWMy4e8Uidra0s/3AybDLEREZkYJ+HH55WS0zihM8oq6WIpLDFPTjUFqYYM3183nytcO0nNS96kUkNynox+muGxdiZnztJ/vDLkVEJCMF/TjVzixh1ZVz+OYrB+jq1b3qRST3KOgnwNqVCznd3c+/bX8n7FJERN5DQT8Bls2fxTV1FXz5xbcZHFRXSxHJLQr6CWCW6mq5t+0MP9qdOw82FxEBBf2Eue3qudTOLOEPvr2TpiMdYZcjInKOgn6CFCZifGXtcmIGax5+idcP6X71IpIbFPQTaHF1OY9/+kaKEjHWPPwSO1tOhV2SiIiCfqItqizjW5++kWRxgk88/LJujyAioVPQT4J5s0t5/NM3Mru8kDu/9DJb950IuyQRyWNZBb2ZrTKzJjNrNrMHMsx/n5n9xMx6zOz3x7JsVNXOLOHxdTdSU1HMXf/8ClveOhZ2SSKSp0YNejOLAw8BtwJLgTVmtnRYsxPAbwN/dRHLRtacimIeW3cDdbNKuPvLW3lBXS9FJATZ7NEvB5rdfa+79wKPAavTG7j7UXffCvSNddmoq06mwv6yqnLu/UojP3izNeySRCTPZBP0tcDBtPGWYFo2sl7WzNaZWaOZNba1RWvP95LyIr75m9dzxZwkn/7aNp7edSTskkQkj2QT9JZhWrbX+We9rLtvdPcGd2+oqqrK8sdPHzNLC/mXe6/nyksruO/r2/n+Tj2CUESmRjZB3wLMSxuvAw5l+fPHs2zkVJQU8LV7lnPt/Jl85pvb+fef6iZoIjL5sgn6rUC9mS0ys0LgDmBTlj9/PMtGUrK4gEfvXs71iy7hf3xrB99qPDj6QiIi4zBq0Lt7P3A/8DTwBvAtd99lZuvNbD2Amc0xsxbg94A/MbMWM5sx0rKT9ctMF2VFCR751AdZubiS//XtnXzj5QNhlyQiEWbuuXdb3YaGBm9sbAy7jEnX3TfAb319Oz948yj/+/Yr+eRNC8MuSUSmKTPb5u4NmebpytgQFRfE2fAb13HL0hr+dNMuHn5hb9gliUgEKehDVpiI8dAnlvGLV8/lzze/wd8806SHl4jIhEqEXYBAQTzG393xAUoL4zz4g2Z2t3by1796DWVF+ucRkfHTHn2OSMRjfOFX3s+f/OLP8czrR/jlL27h4ImusMsSkQhQ0OcQM+PeD13Go3cv59Cps9z+Dz/WzdBEZNwU9Dno5iVVfPf+lVxSXsSd//wKX9myj1zsHSUi04OCPkctqizjid+6iY9cUcWfbtrFH33nVXr7B8MuS0SmIQV9DksWF7Dxzgbu+8jlPLb1IL/+8Eu0dfSEXZaITDMK+hwXixl/8F/ex9+vuZbXDrVz+z/8mFdb2sMuS0SmEQX9NPHfrrmUb6+/CQN+ZcMWvrtDN0QTkewo6KeRq2or2PSZlby/roLfeWwHn3/qTQZ0cZWIjEJBP81Ulhfx9XtvYM3y+Xzx+bf4za82crp7+IO9RETOU9BPQ4WJGH/xS1fzuY9dxQu72/j4Qy+yt60z7LJEJEcp6KexO29YwL/cez0nu/pY/dCLPN90NOySRCQHKeinuRsuu4Tv3reC2pklrH10K3/x5BucPNMbdlkikkMU9BEwb3Yp3/mtm/jYtbVsfGEvH/rCD/mrp5s41aXAFxE9eCRymo508OBze/j+q4cpL0pw94qF3LvyMipKC8IuTUQm0YUePKKgj6g3j5zmwef2sPnVIySDwL9HgS8SWQr6PPbG4VTgP/laEPgrF3HPykVUlCjwRaJEQS+8figV+E/tOkKyOMHaFYtYq8AXiQwFvZyz61A7Dz63h6d3tTKjOME9Ky/j7pULmVGswBeZzsb9cHAzW2VmTWbWbGYPZJhvZvZgMH+nmS1Lm7fPzF41sx1mpvQO2ZWXVvBPdzbwvc+s5PrLLuFvn93Nyr/8AQ8+t4cOXWErEkmj7tGbWRzYDXwUaAG2Amvc/fW0NrcBnwFuA64H/s7drw/m7QMa3D3rRyVpj37qvPZOO//32T08+0YrFSUF3HnDAu5YPo+6WaVhlyYiYzDePfrlQLO773X3XuAxYPWwNquBr3rKS8BMM5s7rqplSlxVW8GXPtnA/7t/JcsXzeah55v50Bd+yNpHt/Ls6626aZpIBCSyaFMLHEwbbyG11z5am1rgMODAM2bmwD+5+8ZMH2Jm64B1APPnz8+qeJk4V9dV8PBdDbSc7OLxrQd5bOtB7v1qI5dWFPNrH5zPr31wHnMqisMuU0QuQjZ79JZh2vDdvAu1WeHuy4BbgfvM7OZMH+LuG929wd0bqqqqsihLJkPdrFL+5y1XsOWBX2DDbyzj8upy/vbZ3az4/A9Y99VGfrS7jUHt5YtMK9ns0bcA89LG64BD2bZx96H3o2b2BKlDQS9cbMEyNQriMVZdNZdVV81l37EzfHPrAb7d2MIzr7cyb3YJa5bP579fN4+qZFHYpYrIKLLZo98K1JvZIjMrBO4ANg1rswm4K+h9cwPQ7u6HzazMzJIAZlYG3AK8NoH1yxRYWFnGH936c2z5o1/gwTXXUjuzhC881cRNf/kc931jO1veOkYudtMVkZRR9+jdvd/M7geeBuLAI+6+y8zWB/M3AJtJ9bhpBrqAu4PFa4AnzGzos77h7k9N+G8hU6IoEef2ay7l9msupfloJ994+QD/tr2F7+88zGWVZfz69fNZ/YFa7eWL5BhdMCXj0t03wPd3HuYbrxxg2/6TAFxVO4MPL6ni56+o5tp5M0nEdZNUkcmmK2NlSjQd6eDZN1p5vuko2w+cYmDQSRYn+FB9JR9eUsWHl1Sr547IJFHQy5RrP9vHi83H+FFTGz/a3caR090AvG9OMhX6V1TRsGA2hQnt7YtMBAW9hMrdaWrt4EdNbTzf1Ebj/hP0DThlhXFuWjy0t1/FvNm6GlfkYinoJad09vTzk7eO83zTUZ5vauOdU2cBuLyqjBWLK1k2fxbXLZhF3awSghP5IjIKBb3kLHdn77EzPB8c4tm27wRnegcAqCwv4roFM7luwSyWzZ/FVbUVFBfEQ65YJDddKOizuWBKZNKYGZdXlXN5VTn3rFzEwKDTdKSDbQdO8tP9J9l24CRP72oFoCBuXHlpBdctmHUu/HVyV2R02qOXnHess4ftQej/dP8pftZyip7+QQBqZ5Zw7fzze/0/N3eGTvBKXtIevUxrleVF3HLlHG65cg4Avf2DvHH4NNuC8N++/yTf23kYgETMWFRZxpKaZPAqZ8mcJAtml6o/v+Qt7dFLJBxuP8v2/ad4/XA7TUc62XO0gwMnuhj6ehcmYlxeVZ4K/pokVwQbgrpZJcRiOuEr059OxkpeOts7QPPRTppaO9g99DrSwaH27nNtSgri1Afhv6Qmda5gwSWl1M0q1YlfmVZ06EbyUklhnKvrKri6ruJd009397GntZM9rR00tXawp7WTH+1u49vbWs61MYO5M4qZf0kpC2aXpd7ThvVQdZlOFPSSd2YUF5zruZPu5Jle9h47w4ETZ9h/vIsDx7vYf6KL595s5Vhn77vaziotYP4lZSyYHWwALikL/hIooTpZTFyHgySHKOhFArPKCrmurPA9GwBIXeR14HjXuY3A/hNd7D9+hu0HTvK9nYdIfxZLPGZUJ4uYU1HM3Ipi5swoSb0PjVcUUzOjmAKdHJYpoqAXyUJ5UYKll85g6aUz3jOvt3+Qd06dZf/xM7xz6ixH2rs53N7NkfZumo508HxTG13BRWBDzKCqvChtA1ASbACKqCxPvaqSRcwqLdRfBzJuCnqRcSpMxFhUWcaiyrKM892djp5+Dp/q5nD7uzcEh0938/axM2x56zgd3f3vWTZmMLusiMryQqqSRVSVF1GZPD8+tFGoLC9idpk2CpKZgl5kkpkZM4oLmDGngCvmJEds19nTz9HT3Rzr7KWto4djnedfbR29tHX2sLftDMc6e85dMJYuZjCrtJCZpQXBeyGzSguYVVZIRUlq2qzSgtT0soJzbYsS6l0UdQp6kRxRXpSgvKqcy6ou3M7d6ezpf+8GoaOH42d6OdXVx8muXt45dZZdh9o52dVLd997NwxDSgvj50J/ZmkByaICZpQkSBYXMKO4gGRxghklwXswXhGMlxcldCHaNKCgF5lmzIxkcQHJ4oIRDxcN1903wMmuXk6e6eNUVy8ng43Bu4f7aD/bR1tHJ6fP9tPR3XfuBnMXUlYYT20USlLBX1Y0/D1+frgwNf1d04oSlBempmmjMTkU9CJ5oLggztyKEuZWlIxpuf6BQTp7+uno7qf9bB8d3f2c7g7e3zXex+mz/efaHmnv5kxPavxM7wADg9ldmFmYiFFaGKe0IE5xYTwYTqSGC1LjmYdTbUoK4hQXxCguOD9clIhTnDY9H3s7KehFZESJeIyZwfH+eRf5M9ydnv7UBuNc+PcMpA2fn9bV28/ZvgG6egc42zsQDPfT3tXLkfdMH/2vjUziMaM4EQvCP05RQYzixPkNQVEitXEoKohRGI9RFGwshqYXJmKp4bTpQ9POvcdT7QoTMQrilpqeNm2qT5or6EVkUpnZuVCtLC+asJ/r7nT3DZ7bGAxtALr7BunuG0i9+lPDPenT+9PbpLdNjXd099PbP0hP/wA9/YOpV98AvQOD9A1MzC1jYpb666UwHqMw2FgUxI3qZDHfWn/jhHxGuqyC3sxWAX8HxIEvuftfDptvwfzbgC7gU+6+PZtlRUQuhplRUhinpDDO7LLCKfnMgUF/10ZgaLi7L9gg9A/QN5Bq09s/SO/AAH39Ts/A4Plp/YP0DQzSOzC0/PlppYWT0wNq1KA3szjwEPBRoAXYamab3P31tGa3AvXB63rgi8D1WS4rIjItxGPnNy7TSTZnJZYDze6+1917gceA1cParAa+6ikvATPNbG6Wy4qIyCTKJuhrgYNp4y3BtGzaZLMsAGa2zswazayxra0ti7JERCQb2QR9ptPDw89IjNQmm2VTE903unuDuzdUVY1yxYiIiGQtm5OxLfCunlV1wKEs2xRmsayIiEyibPbotwL1ZrbIzAqBO4BNw9psAu6ylBuAdnc/nOWyIiIyiUbdo3f3fjO7H3iaVBfJR9x9l5mtD+ZvADaT6lrZTKp75d0XWnZSfhMREclIz4wVEYmACz0zNv9u+iAikmdyco/ezNqA/Re5eCVwbALLmWiqb3xU3/iovvHJ5foWuHvGLos5GfTjYWaNI/35kgtU3/iovvFRfeOT6/WNRIduREQiTkEvIhJxUQz6jWEXMArVNz6qb3xU3/jken0ZRe4YvYiIvFsU9+hFRCSNgl5EJOKmZdCb2SozazKzZjN7IMN8M7MHg/k7zWzZFNc3z8x+aGZvmNkuM/udDG1+3szazWxH8PrsFNe4z8xeDT77PZchh7kOzeyKtPWyw8xOm9nvDmszpevPzB4xs6Nm9lratNlm9h9mtid4nzXCshf8vk5iff/HzN4M/v2eMLOZIyx7we/CJNb3Z2b2Ttq/4W0jLBvW+ns8rbZ9ZrZjhGUnff2Nm7tPqxepe+a8BVxG6u6YPwOWDmtzG/Akqdsk3wC8PMU1zgWWBcNJYHeGGn8e+F6I63EfUHmB+aGuw2H/3kdIXQwS2voDbgaWAa+lTfsC8EAw/ADw+RHqv+D3dRLruwVIBMOfz1RfNt+FSazvz4Dfz+LfP5T1N2z+XwOfDWv9jfc1Hffox/PEqynh7oc9eGauu3cAbzDCA1dyWKjrMM1/At5y94u9UnpCuPsLwIlhk1cDXwmGvwJ8LMOiU/KUtUz1ufsz7t4fjL5E6jbhoRhh/WUjtPU3xMwM+FXgmxP9uVNlOgb9eJ54NeXMbCFwLfByhtk3mtnPzOxJM7tyaivDgWfMbJuZrcswP1fW4R2M/B8szPUHUOOp23ETvFdnaJMr63Etqb/QMhntuzCZ7g8OLT0ywqGvXFh/HwJa3X3PCPPDXH9ZmY5BP54nXk0pMysH/g34XXc/PWz2dlKHI64B/h749ykub4W7LyP1YPf7zOzmYfNDX4eWeobB7cC/Zpgd9vrLVi6sxz8G+oGvj9BktO/CZPkicDnwAeAwqcMjw4W+/oA1XHhvPqz1l7XpGPTjeeLVlDGzAlIh/3V3/87w+e5+2t07g+HNQIGZVU5Vfe5+KHg/CjxB6k/kdKGvQ1L/cba7e+vwGWGvv0Dr0OGs4P1ohjahrkcz+yTwX4FPeHBAebgsvguTwt1b3X3A3QeBh0f43LDXXwL4JeDxkdqEtf7GYjoG/XieeDUlgmN6/wy84e5/M0KbOUE7zGw5qX+L41NUX5mZJYeGSZ20e21Ys1DXYWDEPakw11+aTcAng+FPAt/N0Ca0p6yZ2SrgD4Hb3b1rhDbZfBcmq770cz4fH+Fzw35K3X8G3nT3lkwzw1x/YxL22eCLeZHqEbKb1Nn4Pw6mrQfWB8MGPBTMfxVomOL6VpL683InsCN43TasxvuBXaR6EbwE3DSF9V0WfO7PghpycR2WkgruirRpoa0/Uhucw0Afqb3Me4BLgOeAPcH77KDtpcDmC31fp6i+ZlLHt4e+gxuG1zfSd2GK6vta8N3aSSq85+bS+gumPzr0nUtrO+Xrb7wv3QJBRCTipuOhGxERGQMFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4v4/gS5gyyThoogAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value Function \n",
      " [0.05073115 0.04645272 0.06346115 0.04434864 0.07489288 0.\n",
      " 0.10590912 0.         0.13101275 0.23708198 0.29223709 0.\n",
      " 0.         0.37187037 0.63459526 0.        ]\n",
      "Policy \n",
      " [0 3 0 3 0 0 0 0 3 1 0 0 0 2 1 0]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkoAAAIYCAYAAACSZcNlAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABb+UlEQVR4nO3dd3gU1eLG8e/ZTSGB9ITepUpTqQoiVRAL9i7qlYv+FHv3XhV7b9eOvSD2jooFAanSpLfQA4FASAgkQJLd8/tjl5CEnZAg2ZXk/TzPPmRnzpw5M7Mz++6Zs4ux1iIiIiIiB3KFugEiIiIi/1QKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBqZozxvyfMWaLMWaXMSbpMNbbxxiTVuz5YmNMn8NVv7/OXcaY5oezzgqu/0RjzPJ/6vqNMU2NMdYYExaEtlhjTItKqLeOMWayMWanMeaZw13/QdYd0tdXaaXPqSNNRdtvjJlojBnu//sSY8zPlde6Eut9zRhzbzDWJUeGahmUjDFrjTEDSk27whgz5TDVXylvGoebMSYceBY42Vpby1qbWVnrsta2s9ZOPMx11rLWrgYwxrxrjHn4cNZfWunjaq39w1rbujLXWZbS6w/0uq4CRgDbgFhr7a2VtZLib8r7FH99SWhZa8dYa08+3PUGuu5ba6+x1j50uNclR65qGZSkSB2gBrA41A0JtWD0usghaQIssfplXHGgc1cqnbW22j2AtcCAUtOuAKYUe14f+ALYCqwBbig2rxswHcgG0oGXgAj/vMmABXKBXcAFQB8gDbgDyPAvcyYwBFgBbAfuKU/9/vkWuAFYje/T9lOAy2FbI4HngU3+x/P+aa38bbT+dk4IsGxT//wR/mXTgVsPVrd/Xh8gLdA+B9zAPcAqYCcwB2gEvAw8U6oN3wE3OWybBVr421cA5Pu35btyHMNRwOfAh0AOMPxQj2uxOtsCE/3LLwbOKDbvXf/2jfNv80zgKIftem/ffgYa+Nd7rf95C3yvF1N8/cAHgBfY7W/fHcWO3+XAenyvlf84rLMHsBlwF5t2FrCgAq/JFv6/JwLDyzi32gC/+LdjOXC+Q5veLXVcB/inPVysTOljsBa4DVgA7AA+AWoUmz8U+Mt/zFcBg4FHAA+wx7+elwJsUxzwPr7X0jrgv/jPuX3bBzwNZOF7rZ3isE13AZ+XmvYC8D//31cCS/2vkdXA1WVsa1H7iu2v4vvmNP+2ZgPTgI7F5t0JbPSvZznQ36G9pwLz/PtrAzAqwPUh4OsLiPK3KQtYAtxevP0B1jUQWOY/bi8Bk/a9jgK8hixwHbASWFOO7W0EfOk/fpn++tv6j7nHf9yzHfbjv4FUfK/Xb4H6pdpxjb8dWfjOcVPsXJ3k355twCdO267HP/sR8gaEZKMPEpTw9bTNAe4DIoDm/ovWIP/8zvjeWML8F4ulFHszD3AB6wMU+usL9594W4GPgBignf+EbV6B+n8HEoHG+MLWcIdtfRCYAdQGUvwXkIf885r66wpzWHbf/LFATaCDv90DylF3H5yD0u3AQqA1vjf8TkASvjfjTex/A0oG8oA6Du0r/kb2LiUvbgc7hqPwvQmf6S8bdYjHdV9QCcd3Mb3Hv75++N6EWhdr33b/NoYBY4CPHbbrX+wPexfje0P/pNi8bw62j0sdvzf829cJ2Au0dVjvKmBgseefAXdV9DVPGUEJ3+toA75AEAYch+9NpJ1Dm0of19LPA+2DP/GF5ER/O6/xz+uG701roP+YNwDaBGpzgG16H/gG3/naFN85d1Wx7SvAd167gf/D9zo2AbanCb7XdKz/uRtf8Ozhf34qcBS+8+Ikf9njHLbVMSj592sG0N2/jsv9+yYS33m3Af8bvn97nEJ7H3znvQvoCGwBzizP6wt4HPjDfxwaAYtwCEr4zvUc4Fx859LN+K6ZZQWlX/x1Rx1ke93AfOA5fK+/GkCvQPUG2I/98L0+j/PX9SIwuVQ7vgfi8V2LtwKD/fPGAv/x77uidepx5D1C3oCQbLTvBNqF75PHvkce+y/m3YH1pZa5G3jHob6bgK+KPQ/0hrob/6d1fBdbC3QvVmbOvgtQOesfXOz5tcBvDsuuAoYUez4IWOv/uynlC0ptik17EnirHHX3wTkoLQeGOqxzKf43a2Ak8EMZx7GsoFTmMcQXlCY71V2B47ovKJ2Ir0fGVWz+WPyfwP3te7PYvCHAMof1HuV/TbqA14Cri63nPeCWg+3jUsevYbFpfwIXOqz3YeDtYq/RXKBJRfcNZQelC4A/StX1OnC/w3pKH9fSzwPtg0tLvV5fK7ae5xzWU6LNxbcJ3xvtXuDoYvOuBiYW277UYvOi/cvWdVjXFGCY/++BwKoyXoNfAzc6bGtZQelV/B9ais1fji98tcAXKgYA4WWdAwHa8/y+fXiw1xe+DybFr1MjcA5Kw4AZxZ4bfL3wZQWlfsWel7W9x+MLMAdc50rXG2A/vgU8WWxeLXyhuGmxdvQqNv9T9n+4eB8YXXz/6HFkPqrzGKUzrbXx+x74wsY+TYD6xpjsfQ98PQV1AIwxrYwx3xtjNhtjcoBH8X0iKkumtdbj/3u3/98txebvxncSlrf+DcX+XofvE3Qg9f3zy1PWidO6DrXuRvhCViDvAZf6/74U3y2lQ1HmMfQrvl2Helz3qQ9ssNZ6i01bh6/XYp/Nxf7Ow3+8S7PWrsIX5I/BF8C+BzYZY1rju/BPKmebKrRefD2cZxtjIoGzgbnW2nXwt/dNcU2A7qWOyyVA3UOoy4nT9pb1uitLMr5ewtKv9YDH1lqb5/+zrP18kf/vi/3PATDGnGKMmWGM2e7fN0M49P18a6n93AhfL1IqvqA7CsgwxnxsjAl43hpjuhtjfjfGbDXG7MB3m6l0e5z2d30OvHY4KVHW+pLGBufiUGq+4/b6/11nrS08SH1O7Spqt7V2F75bd+U5r+/AF/j+9H/r91+HsH75B6jOQaksG/Dd944v9oix1g7xz38V3730ltbaWHxvwOYwrr889Tcq9ndjfF39gWzCdxEpT1knTus61Lo34Os1CeRDYKgxphO+MQRfl7ONNsA6yjqGgZb5O8d1E9DIGFP8nGqMbxzIoZiE7zZEhLV2o//5MCAB3ziMQEpvT4VYa5fge1M4hVJv4FRs3+Ti61XZp3gI2gBMKnVcallr/6+czSyr7oMp63VX1r7bhq8XofRr/VCP7WdAH2NMQ3zjwD4C8AfUL/CNdarj/wD3A877OY+y9/MjpfZztLV2LIC19iNrbS//NlngCYd1fIRvXE4ja20cvh7O8p4T6Rx47ShXWWOMKbVsIMWPWVnbuwFo7DDo+2DnTIlrnDGmJr5hAgc99tbazdbaf1tr6+PrgXzlSPg2tBxIQSmwP4EcY8ydxpgoY4zbGNPeGNPVPz8G3/30XcaYNvjGJBS3Bd+YmEN1sPoBbjfGJBhjGgE34hu0GshY4L/GmBRjTDK+MTsfVrA99xpjoo0x7fCNLdm3rkOt+03gIWNMS+PTcd9vOFlr04BZ+HqSvrDW7i6romJK7/ODHcNA/s5xnYnvTfwOY0y48f1m1OnAx+Vsf2mT8N16nOx/PhG4Ht9tAo/DMn/3dQe+N8YbgN743tD3Kc9rcp+/8PVMRfvfGK4qNu97oJUx5jL/fgo3xnQ1xrQtZ/v+AoYYYxKNMXXx9YyU11vAlcaY/sYYlzGmgX9boIx959/fnwKPGGNijDFNgFuo+Hm0r76t+I7nO/jC/FL/rAh842C2AoXGmFOAsr4S/xdwsf+1PRhfb+M+bwDX+HuEjDGmpjHmVH/7Wxtj+vmD2R58vdlOr6kYYLu1do8xphu+AF1enwJ3+69TDfG9fp2MA9oZY872B5obqFgIdtxefNeCdOBx//Qaxpie/uW2AA2NMREO9X6E7zVzjH9/PQrMtNauPViDjDHn+bcbfAO9Lc77Wf7BFJQC8F8YT8d362MNvk+Ub+L75gv4vlVzMb7Bum9wYEgZBbzn7wI+/xCacLD6wTewdA6+i+U4fG8CgTwMzMb3LaCFwFz/tIqYhG+g8m/A09bafT/8dqh1P4vvIvozvjfft/ANyNznPXwDSCty2+0t4Gj/Pv+6HMcwkEM+rtbafOAMfL0x24BX8I1DWVaBbShuEr43qX1BaQq+3oPJjkvAY/iCa7Yx5rZDXO9YfGNhJlhrtxWbXp7X5D7P4fum2hZ8x3LMvhnW2p343vwvxPdpfTO+3ozIcrbvA3wDc9fie/2U1Y4SrLV/4gv6z+Eb1D2J/b0FLwDnGmOyjDH/C7D49fiC8Gp8x+Ij4O3yrjuAj/CNESrqtfPvmxvwnRtZ+Pb3t2XUcSO+13g2vtuXXxeraza+weUv+etKxTceB3z7+nF8r9PN+L6McY/DOq4FHjTG7MT3QejT8m4g8AC+Hso1+I6V4/nsf62d529XJtASmFreFZW1vcWuBS3wfTsvDd9YOYAJ+L6hutkYs41SrLW/Affi6+lLx9cjeWE5m9UVmGmM2YXvON5orV1T3m2Sf459X2OUI4gxxuK7BZJayetpiu8iF36I9/cPdb298X1ab1pqzI+IiEhQqUdJ/lGM79fCb8T3DTGFJBERCSkFJfnH8I9TyQbq4fsasoiISEjp1puIiIiIA/UoiYiIiDhQUBIRERFxEIz/ddm+s2BREFYjoXRlx/a8v2hxqJshlWxY+3YAvDJ7YYhbIpXp2i4dAHROVwPD2rc7nD+WXCWpR0lERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcRAW6gaE0up58/j1nbfxer106t+f4886u8R8ay2/vvM2q+bOJTwyglOvu566zZsD8Mq11xBZIwrjcuFyu7niiScB2L1zJ9889yw7tmYQl1KbM2+5lRq1arH4j8nM/Oaboroz1q/jyieeok6zZnzy8EPsys7Cejw0bHs0J181HJfbzfoli/nt3XfIWLeOoTfdQpvjjw/ezqlCVs2by89vv431ejmm/wBOOPvA4/zz22/5jnNEJKddP5J6zY8qmu/1eHj7zjuISUzkgnv+UzR91g/jmP3jj7hcblp07kz/YcOK5u3YupXXb7qR3uefT4+hZwLwwX33sisri7CICAAuvu8+asbFM2f8eOb89CPG5SKiRg2GXPN/pDRqVIl7pOpaO38ekz54B+v10q5Pf7qecVaJ+dZaJr3/NmvnzyMsIoKTrx5J7Wa+c3pvbi6/vvEqmWnrwRgGjriWei1bM/2zsayaMwtjXETHxjLwmpHUSkgsqjNn21Y+vONmup9zHp1PHQqAp7CAie++RdrSxRhjOP78i2nZrQcbly5h0ofvsG39Ok4ZeTMtu+ucPhSVdU4DzPjma357/31ufuddomNj2bhyJT+89uq+ijnxggto071HiWU+fexRsrdsYcTzLwCwIyOD7195mbwdOdSIqcXQG28kNim5EvaEBEO1DUpej4ef33qDC++9j5jEJN69+05adulKcrE3qNXz5pKVns7VL77EppUrGf/GaC5/7PGi+ReNeoDo2NgS9c74+iuadOjA8WedzfSvvmT611/R99LLaHdib9qd2BuAjHXr+OLJx6nTrBkAZ95yK5HR0Vhr+eqZp1g2YzpH9+xFbHIKp143kpnffhuEPVI1eT0efnrjDS6+735ik5J4+847aNm1a4kgsmruXLanp/N/L73MppUr+Gn0aK58/Imi+bPGjSO5QUP27s4rmrZ24UJW/DmLfz/7HGHh4eTuyC6x3l/eeYejjj32gPYMvfEm6rdoUWJa+xNPpPOgQQCsmPUnv777Dhfde9/h2Pxqxev1MPHdNznr7vuolZjIx/feRfPjupDUcP+xXjt/Htmb07n8mRfZnLqSCe+M5sIHfef0pA/epkmnYzj1ptvwFBZQuDcfgONOHcrx510EwF8/jWPml5/R/6qri+qc/OG7NOl0TIm2/Pn1l0TFxnH5My9ivV725O4CICY5mYFXX8fccTqnD1VlndMAOdu2sWb+AmKT94ea2o0bc9WTT+Fyu9mZtZ03b7mFVl264nK7AVg2YwYRNaJK1PPr++/R4aQ+dOzbl7ULF/L7h2MYeuONlbE7JAiq7a239NRUEurWJb5OXdzh4RzdsxcrZ88qUWblrFm0P+kkjDE0aNWKvbm57MrKKrPelbNm0aFPXwA69OnLyj//PKDM0qlTOLpnr6LnkdHRgO8C4CksxGAAiK9dm9pNmmKM+VvbWp1tSk0lsW49Eur6j3OvXqyYVfKYrJj1Jx1P6uM/zq3Zk5vLzqztAORkbiN17hyOGTCgxDJzx4/nhLPOIiw8HICacfFF85bPnElCnTrl7hXad/wBCvbsBR3vQ7JlVSpxdeoSV7sO7rBwWvXoyeo5Jc/p1XNm0fZE37Gu17IVe/PyyM3KYm9eHhuXLaVdn/4AuMPCiaxZEyh1fPbuLXE+rpr9J3G165QIYwBLJk0o6s0yLhdRMb4PVLEptUlp3BRjqu2l92+rrHMa4Jd33qbfsMtKHOPwyMiiUOTJLygxL3/3bmZ+9y09zz23RD3bNqTRtGMHAJq0b39A++TIctAeJWNMG2Ao0ACwwCbgW2vt0kpuW6XauX07McW6QmMSE9m0cmXZZZKS2Lk9k1oJCRgMnzz8IGA4duBAjhl4MgC5O7KplZAAQK2EBHJzdhyw7qXTpnLOHXeVmPbJww+yKTWVo445ltY9ehywjByandsziUlOKnoem5jExgDHufgnyNikJHZmbicmIZFf3n6bfpcNI3/37hLLZKZvYv3SpUwc+xFh4eH0v/xy6rdoSf6ePUz/+isuvu9+Znz7DaV9//JLGJeLNj160Ovc84ouurN//JGZ332Lp7CQS0c9cDh3QbWxq9T5Wisxic2rVpYqk0mtpKRiZRLZlZWJy+UmKiaWX15/mW3r11K72VGcdNmVhNeoAcC0Tz9i6R+TiIyO5uz/jAKgYM8eZn/3NWfdfW+JHqK9ubkATP/8Y9KWLiaudh36XDG8RJiWQ1dZ5/SKWX8Sk5hEnabNDljnxhUr+P7ll9mxbStn3HBDUXCa9PFYup9xBuGRkSXK12nalGXTZ9DttNNYPnMm+bt3k7dzJ9ExMX97+yX4yvxYY4y5E/gYMMCfwCz/32ONMXeVtew/nz1wUulP8vbAMvt6ey59+BGufPJpzv/Pf5kz/ifWL1lcrrVuWrmC8IhIUho3LjH9gv/ex/Wj36SwsIB1ixaVbxPk4Mp1mAMcZ2NYOXs20XFx1DvqqAPmW4+HPbm7uOKxx+k37HK+fOYZrLVM/uRjup12OhFRUQcsM/TGmxjx3PMMe/gRNixdysJJE4vmdTnlFK575VX6XXYZU774vMKbKRDoYJerN9YYvF4PGWtX03HAyVz86NOER0Yy+7uvioqccP7FXPXi67Q+4UTm//wTADO++IRjTzntgNsuXq+HXdszqdeqDRc/8hT1WrZmypj3/96myX6VcE4X7N3L1C++oPeFFwZcZYNWrbj6hRf41xNPMu3LLynMz2fzmjVsT998wHglgP6XX876JYt587ZbWb94MTGJibhc6kU8Uh2sR+kqoJ21tqD4RGPMs8Bi4PFACxljRgAjAF5//XXCe5xwGJp6eMUkJrEzc1vR853btxOTmFiyTFKpMpmZ1PKX2Ve2Zlwcrbp1Jz01lcZHt6NmXDy7srKolZDArqwsasbGlahzydSptO3Vi0DCIiJo2aUrK2f9SbNOnQ7LdlZ3MUlJ7NyWWfQ8Z/v+Y7hPbFISOdv2H+eczExqJSawbPp0Vs6axaq5cyksKGBvXh7fvPA8Q2+8iZikJNp07+Hr2m/ZEmMMeTk5bFq5kmXTpzPhg/fZk5uLcblwh0fQdcgQYv09GZFRUbTrdSKbVqbS0X+bdp92PXvx0+jRlbhHqq5apc7pXdszqRmfcECZXZmZxcpsp1Z8IhjfvLotWgHQolsPZn/39QHraH3CiXz79KMcf+4FbF61kpV/zmDK2A/Ym5eLMS7CwiPoOHAwYZGRtOjSDYCW3Y9n8cTfKmGLq6fKOKePP/Mssrds4c1bbykq/9btt3Hl408U3SEASG7YkIjIGmSsX096aiqbV6/ipWuuxuvxkJuTwwf33ctlDz5ETGIi595xJ+C7PbdsxnRq+G/lypHnYEHJC9QH1pWaXs8/LyBr7Whg39XevrPgn9dDUq9FC7anp5O9ZQsxiYksmTqFM268qUSZFl26MvenH2nbsxebVq4kMjqaWgkJ5O/Zg7WWyKgo8vfsYe38+fQ89zz/Ml1YOPF3jj/rbBZO/J2WXbsW1We9XpZPn8YlDz5UNC1/927y9+yhVkICXo+HVXPn0qht26Dsg+qgfunjPGUKZ950c4kyLbt2ZfaPP3J0r15sWrmCyOhoYhIS6XvppfS99FIA1i1axIxvv2Go/zXSqlt31i5cSJP27cnctAlPYSHRsbEMe/iRononf/IxETVq0HXIELweD3tyc4mOjcVTWEjqnNk07egLw9s3bSKxfn0AVs6ZQ0K9ekHYM1VPneYtyN6czo6MLdRKTGTFjKkMvu6mEmWaH9eF+T//SKvje7I5dSWRUdHU9L8RxiQlkbVpIwn1G7Bh8UISGzQEIGtzOgl1fcdk9dxZJNRrAMB59z1cVO+MLz4hvEYNOp18im89x3YmbeliGrXrwIZF++uSv6+yzumb33m3aPmXrrmafz35FNGxsWRv2UJscjIut5sdGRlkbtpIfO3a1G/Rgs6DBwOQnZHBp48+wmX+a3teTg5RtWphXC6mfvklnfr1r/wdI5XmYEHpJuA3Y8xKYIN/WmOgBTCyEttV6VxuNydfNZxPHnkI6/XSsW8/Uho1Zt7P4wE49uRBHHXccayeN5fXr7+O8IhIhlx3HQB5O7L54infzwFYj4eje51Ic/83nI4/62y+fvYZFkz4jdjkFM685daida5fuoSYpCTi69Qtmlawdy+fP/EYnoICrNdL4/YdOPZk3zeg0lNT+fKpJ9iTm0vqnNlM+fRjhj/3QlD2T1XhcrsZNHw4Yx960PczEP36k9K4MXPG+45z50GDaHFcZ1bNncsr111LeGQkp1138Jf2Mf368f0rLzP6phtxhYVxxvU3lHmbp7CgwNeGQg9er5dmHTtyrH8w6ewff2TNggW4wtxE1azFGSOvPzwbX8243G76XDGcr594GOv1cvRJ/Uhq2IgFv/qOdccBg2h6zHGs/Wsu790ykrCISAZefW3R8n2GXcVPr7yAp7CQuNp1GHi173yf+vGHZKdvAmOITU6h379GHLQtPS+8jPGv/o9JH7xDVGwsA0f46tq8KpVxzz3Jnrxc1sybzYwvPuGyJ58//DujCqusc9rJhqVLmfbVV7jC3BhjGPzvEQd827m0dYsX8fuHYzAGGh19NIP/ffDXjPxzmUD3cksU8H09oxu+wdwGSANmWWs95VzHP7JHSQ6vKzu25/1F5RunJUeuYe3bAfDK7IUhbolUpmu7+L6xpXO66hvWvp2+ZnsQB/3Wm7XWC8wIQltERERE/lE0DF9ERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoih9FnLzzPZy88H+pmiMhhonNaDvqf4opI+Yx96il+HvMhAIUFBVx02+0hbpGI/B06pwUUlCqdtRZjTKibIZVszm+/sXzuHAZcdDEAy+fMYe6ECRzXr1+IWyaHm87p6kHntOyjW2+VyOvx8PpddzLn119D3RSpZJ379+fut94mOqYW0TG1uPutt3VBrYK8Hg/v/PdO/pqgc7qq0zkt+6hHqZJ4PR5G330Xe/Jy6di7d6ibI0EQGR0d8G+pGrweD+/eezd78nJp10vndHWgc1pAQanS/Dn+J/4c/xMAV3ftHLDMvx58iF5Dzwxiq0TkUM355Sfm/OI7p2/q2SVgmUvve4jjTx8azGaJSCVTUKok3QYNZt7vv7N3925GPvc8YeHhoW6SiPwNnQcOZsGkieTv2c2/n3xO57RINaExSpXE5XZz9eNPUCM6mr8mTQx1c0Tkb3K53Vz58ONERkWz8I+JIW6NiASLepQqkcvt5uonntQ3ZESqCJfbzZWPPKFzWqQaUVCqZLqgVi9nXXtdqJsglUzndPWic1p0601ERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiANjra3sdVT6CkREROSQmFA34J8uLNQNEBERkWpo8RcV60hpd05IQl1QgtKz0+cHYzUSQrcc34kJm1ND3QypZP3qtgDg7p/nhbglUpkeO/lYAJ3T1cC+czoUrMdTofKh6vpSj5KIiIgEn6cw1C0oFwUlERERCTrrrVhQUo+SiIiIVB8VvPUWKgpKIiIiEnRWt95EREREHCgoiYiIiARW0TFKoaKgJCIiIsGnMUoiIiIigWmMkoiIiIgTBSURERGRwKxXt95EREREAtKtNxEREREnCkoiIiIigenWm4iIiIgT9SiJiIiIBKYxSiIiIiJOFJREREREAjtSxii5Qt0AERERqX5s/t4KPQ7GGPO2MSbDGLPIYb4xxvzPGJNqjFlgjDmuPO1UUBIREZGgs57CCj3K4V1gcBnzTwFa+h8jgFfLU6luvYmIiEjweQ/vGCVr7WRjTNMyigwF3rfWWmCGMSbeGFPPWpteVr0KSiIiIhJ01lOxMUrGmBH4eoL2GW2tHV2BKhoAG4o9T/NPU1ASERGRf5gKBiV/KKpIMCrNBKr2YAspKImIiEjQheB3lNKARsWeNwQ2HWwhDeYWERGR4PN4Kvb4+74Fhvm//dYD2HGw8UmgHiUREREJgYqOUToYY8xYoA+QbIxJA+4HwgGsta8BPwBDgFQgD7iyPPUqKImIiEjQHe4fnLTWXnSQ+Ra4rqL1KiiJiIhI8B3mHqXKoqAkIiIiQXe4b71VFgUlERERCTrr8Ya6CeWioCQiIiLBp6AkIiIiEphuvYmIiIg4sJ6D/ij2P4KCkoiIiASdxiiJiIiIOFBQEhEREXFgvbr1JiIiIhKQxiiJiIiIOLBHxpfeFJREREQk+NSjJCIiIuLAe2SM5VZQEhERkeDz5oe6BeWjoCQiIiJBpx4lEREREQcazC0iIiLiwOs1oW5CuSgoiYiISNDp1puIiIiIA916ExEREXGgW28iIiIiDrzqURIREREJTD1KIiIiIg6sgpKIiIhIYPrWm4iIiIgD3XoTERERcaCgJCIiIuLAo6AkIiIiEph6lEREREQceK2C0j/e+gV/Me2jd7BeL2169+fY084sMd9ay7Qx77B+wTzCIiLpM/xaUpo2Jzt9E7++8lxRuZytGXQ563w6Djq1aNr8H79lxicfMuzFN4mKicVTWMjkd15j27o1eD1eWvXszbGnnQVA6sxpzPvuS6zXS+NOx9HjgksBWDLhZxZPGI8xLsJr1KD3FVeT0KBh5e+YKmbxzNl8+uJorNdLz1NPZtAl55eYb63l0/+9zuKZs4mIjGTY3TfTuFWLMpfNzdnJm6MeJ3NzBkl1azP8gbuoGRMDwE8ffsq0H37GuFxccMPVHN2tMwDrlq/k/ceeoyA/n3bdu3D+DVdjjKEgv4D3Hn2G9StSqRkbw/D77yKpXp0g7qGqY/OSv1jwxXtYr5emx/ej9clDS8zfuXkjc8a8RnbaGo4+7QJa9T/9oMsu+eEz1k6bQGStWADanX4hddsdi7ewkLkfv0H2+tUYY+h47uWktGwHQNb61cz58FU8BfnUbXcsHc+5HGMMedu3MmfMa+zdtZOI6Jp0GTaS6ISkIO2dqkPndNVwpHzrzRXqBoSK1+tl6gdvMeSWezj/0edInTmVrI1pJcpsWDCPHVs2c+ET/6P3FSOY8v6bAMTXq8+5Dz3FuQ89xdkPPEFYRATNOncrWm5X5jbSFi+kVlJy0bTVs2bgKSjkvIef4exRj7Pk91/ZuTWDPbt2MvOTDzjtjvs4/9Fn2Z2TTdqShQC0OL4X5z38DOc+9BSdhgxl2tj3grBnqhavx8PHz7/KyCcf4L73XmXWb5NJX7u+RJnFM2eTkbaJB8a8wcW3Xc/YZ18+6LLjx3xGm86dePCjN2jTuRM/j/kMgPS165k9YTL3vvsq1z/1IGOfewWvx/fzs2OffYVLbrueB8a8QUbaJhbPnAPAtHHjiY6pxYMfvUm/887kq9ffCdbuqVKs18v8z96m5//dxcD/PEPanKnkpJc8p8Nr1qLjuVfQst9pFVq2Rd8h9L/rCfrf9QR12x0LwJppvwEw4J6n6DnyPyz86kOs/8r/1ydvcexF/+bk+55nV0Y6W5b8BcDCrz6kcbfeDLj7SdoMPofF342trN1RZemcrjo81lToESrVNihlrE4ltk5dYmvXwR0WRovuJ7B23qwSZdbOm02rnr0xxlCnRSv25uWSm51VoszGJQuJrV2XmOSUomnTxr5Hj/MvAfYfWGOgYO8evB4PnoJ83GFhhEdFk5Oxhbi69YmK9X1abXB0R9bMnglARFR00fKFe/dgzJHRTflPsnbpClIa1Celfj3CwsPp0q8386fMKFFm/pQZ9BjUD2MMzdu1IW9XLjsyt5e57PypM+gxeAAAPQYP4K9906fMoEu/3oRHhJNcry4pDeqzdukKdmRuZ09eHs3bt8UYQ49B/Zg/Zbq/rpn0GNQfgONO6sWyufOx1gZrF1UZ29elUjO5LjWT6+AKC6Nh5xNIXzi7RJkaMXEkNjkK43ZXeNnSdm7eSO3W7YvqDY+KJmv9anbvyKJgz26SmrXCGEPjbr3Z5K8rZ/NGUlr5lklp1Y70hXMO1+ZXGzqnqw6v11ToESrVNijlZW2nVuL+Lu+aCUnkZm0vUSY3azs1E5NLlMkrVWbVzKm06NGz6PnaebOpmZBIUuOmJco169KD8MgafHDTCMbcci0dTzmdGrVqEVenLtnpG9m5NQOvx8PauX+yK3Nb0XKLfv2Jsbdfz4xPx9DzkisPx6ZXK9nbMkmovf8YJqQkk70tM0CZlJJltmaWuezOrGzikhIBiEtKZGdWtsP6ksjelkn21kziU/a/3uKL1VV8/e4wN1E1o8ndkXM4Nr9a2ZO9nahit7Gi4hPZnb29jCXKv+zqyeP59bE7mDPmNfLzdgEQ16Ax6Qtm4/V4yN2WQfaGNezOzmTPju1ExSeWqGuPv664Bo3ZNN/3QWjT/FkU7tnN3tydh77R1ZDO6arjSOlRqrZjlAKne1O6UIAi+8t4CgtZN28O3c69GICCvXuZ992XDLntvwcstnVNKsbl4tLnXic/L5dvHr2Phkd3ILZ2HXoNG86vrz4PxlC3RWtytm4pWq79gMG0HzCYldOnMPe7L+j775GHsrnVVqDjfMDpFvA4l3PZcqwPYxzqMmWsX72HFRXw83o592NZyzbvNZC2g88BYMm4T1n41Yd0vuQamvToy87NG/n9qXuITkwmsVkrjMsd8HDuq6vDWZcy/9N3WDdzMslHtaFGfCIulzvAAuJE53TVocHc/3A1E5PYtX3/p5DcrExqJiQcUCZ3+7YSZaLj95fZsGAeyU2aER0XD0BOxhZytmbw+b23F5X/8v47Oeu+x1g5fQqNOhyDOyyMqNg46rZszda1q4itXYemx3ah6bFdAFgy8VeM68COvhbdT2DK+28ctu2vLhJSksnK2H8Ms7ZuIy655ODZ+JRksjK2ligTn5yEp6DQcdmYhHh2ZG4nLimRHZnbiUmId1hfJvFJiSTU9n2i3Sd76zbikhNLrD+hdjKeQg+7c/OoGRtz+HZCNREVn8jurP37eHf2dqLiEspYonzL1oiNL5re9IR+TH/9SQBcbjcdz7m8aN7EZ++lVkpdwqNrluiN2p29nRr+uqLiEunx71sB3+30jfP/JLzYLXY5OJ3TVUf+EfLzANX21lvtZkexY0s6OVsz8BQWkjpzGk38YWWfJsd0YcXUyVhr2ZK6goioaGoWC0qpM6ZyVLHbbkmNGnP5i29yyTMvc8kzL1MzIYmzH3iC6Ph4YpKS2bh0EdZaCvbuYcuqlcTXawDA7pwdAOzN3cWS38bT5qR+AOzYnF5U97r5c4mtU6/S9kdV1aRNKzLSNrItfTOFBQXMnjCZjj27lyjTsWd3ZoyfgLWW1YuXEVWzJnFJiWUu27Fnd2b89CsAM376lU49exRNnz1hMgX5BWxL30xG2kaatm1FXFIiNaKiWL14GdZaZoyfQKdePYqt3zcweO6kKbQ+tqPGox2ChMZHsWvrZnK3ZeAtLCRtzjTqdej8t5fdvWP/uMRN82cRW68RAIX5eyncuweALcsWYFxuYus1JCougbAaNdi+ZiXWWtb/OZn6HXzXlr27cooGfC//+Wua9uhzuDa/2tA5XXV4ranQozyMMYONMcuNManGmLsCzI8zxnxnjJlvjFlsjDnomJZq26Pkcrvpdem/+OHpR7BeL61P7Etig0YsmfAzAEf3O5nGnY5l/YK5fHzHDYRFRtDnqmuLli/Yu5e0xQs48YoR5Vpfu/6DmfjmK3z2n1uxWFr36ktSoyYATB3zDpkb1gHQ+Yxzia9bH4BFv/3ExsULcbndRNasRd9/X3c4d0G14A5zc+FN/8eLt92L1+vlhCEDqd+sCZO/+QGA3kOH0L5HVxbNmM19Fw/3fZX4rpvLXBZg0MXn8eaox5k67hcS66Tw7wfuBqB+syZ07tuLBy+/BpfbzYU3XYvLP3D4oluu473Hn6Ng717ade9Cu+6+N8+eQ07m3Uee5r6LhxMdE8NV998R7N1UJbjcbo4570qmvvIo1npp0qMvsfUasXrKL4DvFtqenGwmPHUPhXt2Y4whdeKPDLznacKjogMuC7DomzHsSFsHxhCdmMKxFw4HYO/OHUx95TGMMdSIS6TrsP3n57EXXFX08wB12h5DnaOPAWDbyiUs+u5jDJDUoi3HnPevoO6jqkDndNVxuMcdGWPcwMvAQCANmGWM+dZau6RYseuAJdba040xKcByY8wYa22+Y71BGIlvn50+v7LXISF2y/GdmLA5NdTNkErWr67vt2ju/nleiFsilemxk30/gaBzuurrV7dFyLq6Pj92QIUCyLnzfi2zrcaY44FR1tpB/ud3A1hrHytW5m6gEb7A1BT4BWhlrXX8Vadq26MkIiIioeM56FD6CmsAbCj2PA3oXqrMS8C3wCYgBrigrJAE1XiMkoiIiISOx1bsYYwZYYyZXexReuxLoORVutdqEPAXUB84BnjJGBNbVjvVoyQiIiJB56lgeWvtaGB0GUXS8N1W26chvp6j4q4EHre+cUepxpg1QBvgT6dK1aMkIiIiQeep4KMcZgEtjTHNjDERwIX4brMVtx7oD2CMqQO0BlaXVal6lERERCToDvcYJWttoTFmJDAecANvW2sXG2Ou8c9/DXgIeNcYsxDfrbo7rbXbHCtFQUlERERCwFMJ37q31v4A/FBq2mvF/t4EnFyROhWUREREJOgqOkYpVBSUREREJOgUlEREREQcKCiJiIiIOPAc8BNH/0wKSiIiIhJ06lESERERcVAZ33qrDApKIiIiEnTqURIArLUYE7L/nFmCRMe5+tCxrh50nCvfkTJGSf+FSSXyejx8eP/dLJj4W6ibIpXI4/HwyLU388e48aFuilQyr8fDt4/fw/IpOqerMp3TweHBVugRKupRqiRej4cxo+5hb14uR59wYqibI5XE4/Hw6HW3snvXLroP6BPq5kgl8no8fPfEf8jfncdR3XROV1U6p4NHt96qub9+G89fv/k+jdzZp1vAMhf85wG6nTo0mM2Sw2ziN+OY+M04AE5p2j5gmduff5zBF5wTzGZJJVg66WeWTvoZgKdP7xGwzJBbR9Hx5DOC2Sw5zHROB48Gc1dzx/QfxKI/JpK/ezdXPPYsYeHhoW6SVII+Q09l2vjf2J2XxwNvvUx4RESomySVpO1JJ7Ny+kQK9uzm7Puewa1zukrSOR08GqNUzbncbi4d9RiRUdEsmTIp1M2RSuJ2u7nnlWeJqlmT6T9PCHVzpBK53G7OuOtRIqKiWTljcqibI5VE53TwFFhvhR6hoh6lSuRyu7n0wcf1zYkqzu12899Xn9NxrgZcbjdn3P2YjnUVp3M6OI6UHiUFpUqmE6160HGuPnSsqwcd58qnoCQiIiLiQIO5RURERByoR0lERETEgVc9SiIiIiKBqUdJRERExIGCkoiIiIgD3XoTERERcaAeJREREREH+nkAEREREQde9SiJiIiIBKYeJREREREHGswtIiIi4kCDuUVEREQceK031E0oFwUlERERCToN5hYRERFxoMHcIiIiIg7UoyQiIiLiQN96ExEREXFwZAzlVlASERGRECjUt95EREREAtOtNxEREREHR8pgbleoGyAiIiLVjxdboUd5GGMGG2OWG2NSjTF3OZTpY4z5yxiz2Bgz6WB1qkdJREREgs57mDuUjDFu4GVgIJAGzDLGfGutXVKsTDzwCjDYWrveGFP7YPWqR0lERESCrhJ6lLoBqdba1dbafOBjYGipMhcDX1pr1wNYazMOVqmCkoiIiARdRYOSMWaEMWZ2sceIUlU2ADYUe57mn1ZcKyDBGDPRGDPHGDPsYO3UrTcREREJuop+6c1aOxoYXUYRE2ixUs/DgM5AfyAKmG6MmWGtXeFUqYKSiIiIBF0lfOstDWhU7HlDYFOAMtustblArjFmMtAJcAxKuvUmIiIiQWcr+CiHWUBLY0wzY0wEcCHwbaky3wAnGmPCjDHRQHdgaVmVqkdJREREgu5w9yhZawuNMSOB8YAbeNtau9gYc41//mvW2qXGmJ+ABfj+F5U3rbWLyqpXQUlERESCrjJ+btJa+wPwQ6lpr5V6/hTwVHnrVFASERGRoDsyfpdbQUlERERC4Ej5L0wUlERERCTojoyYpKAkIiIiIXCkBCVjK/qLTxV3pOwLERGR6ibQjzQGRbMmTSqUD9asWxeStqpHSURERILuSOlFCUpQGvH13GCsRkJo9JnH0ePe+aFuhlSyGQ91AmDQ5b+HuCVSmca/1xeACZtTQ9wSqWz96rYIdRP+8dSjJCIiIiEQsrt+FaKgJCIiIiGgoCQiIiLiQEFJREREJDDjCnULykVBSURERILOqEdJRERExIFRUBIREREJSD1KIiIiIo40RklEREQkIKNbbyIiIiIO9K03ERERkcCMbr2JiIiIBKZbbyIiIiJOdOtNREREJDCjoCQiIiISmMYoiYiIiDhQj5KIiIiIA2PcoW5CuSgoiYiISNCpR0lERETEgYKSiIiIiAPdehMRERFxoB4lEREREQfqURIRERFxoKAkIiIi4sClW28iIiIigblcEaFuQrkoKImIiEjQ6dabiIiIiAPjUlASERERCch1hPQoHRkjqURERKRKMcZdoUf56jSDjTHLjTGpxpi7yijX1RjjMcace7A61aMkIiIiQXe4xygZX4UvAwOBNGCWMeZba+2SAOWeAMaXp171KImIiEjQGRNWoUc5dANSrbWrrbX5wMfA0ADlrge+ADLKU6l6lERERCToKmGMUgNgQ7HnaUD34gWMMQ2As4B+QNfyVKqgJCIiIkFX0W+9GWNGACOKTRptrR1dvEiAxWyp588Dd1prPcYEKn4gBSUREREJunLeTiviD0WjyyiSBjQq9rwhsKlUmS7Ax/6QlAwMMcYUWmu/dqpUQUlERESCrhJ+cHIW0NIY0wzYCFwIXFy8gLW22f71m3eB78sKSaCgJCIiIiFQ0R6lg7HWFhpjRuL7NpsbeNtau9gYc41//muHUq+CkoiIiARdZfzgpLX2B+CHUtMCBiRr7RXlqVNBSURERILOuI6MCHJktFJERESqlMN9662yHBmtFBERkSqlEgZzVwoFJREREQk69SiJiIiIONAYJREREREH6lESERERcaKgJCIiIhKYbr2JiIiIODDuGqFuQrkoKImIiEjw6ecBRERERALTrTcRERERJxrMLSIiIhKYVY+SiIiIiAOXxiiJiIiIBKagJCIiIhKYVVASERERCUxBSURERMSJgtI/X7vasVzQoSEuA1PWZfLTyi0l5tetFcnlxzWhcVw0Xy/dxC+pGQCEuQy3n9iKMJfBbQxzNmXz3bJ0ADrXj+f0NvWoG1ODxyYtZ112XlF9g1vWoVeTJLwWPl64gSUZOwE4s219ejRKJDrCzQ3fzy8q37tpMn2bpeDFsrfQywd/rSd9557K3i1VTo8WMdx8an1cxvDtnO188EdGifmDOsZz2Ym1AcjL9/Lkd2mkbt5DRJjh1ataEBFmcLsMExZn8+YE32vk4fOb0Dg5EoCYGm527vEw7JUVAAzrXZvTj0vEay3PjtvEzFTfcR7YIZ7LT6oNFrbuLGDU5+vZkefhohOSOaNzEh6vJSu3kEe+2sDmHQXB2j1VSpcOiVxzSUvcLvhxUjqfjltfYn7f4+tw/qmNAdizx8OL7y1n9YZcAM4c2JBT+tTDGMOPEzfx1c9pAAy/4Ch6HJNEgceSnrGbZ95cRm5eIce1S+Bf5x9FmNtQ6LG88XEq85dmA/DIrR1JjI/E7TYsWp7NS++vwGvh6otb0KlNPACRkW7iY8I559opwdk5VcjimbP59MXRWK+XnqeezKBLzi8x31rLp/97ncUzZxMRGcmwu2+mcasWZS6bm7OTN0c9TubmDJLq1mb4A3dRMyYGgJ8+/JRpP/yMcbm44IarObpbZwDWLV/J+489R0F+Pu26d+H8G67GGENBfgHvPfoM61ekUjM2huH330VSvTpB3ENHButyhboJ5VJtg5IBLu7UiOemriRrdwH39GnN/M07SgSR3HwPHy9I49h68SWWLfRanp2ykr0eL24Dd5zYmkVbdrAmK4+NOXt49c/VXHpM4xLL1IupQdeGCYyasJS4GuHc0rMl//1lMRaYvzmb31dn8NDAdiWW+TNtO5PXbgOgU904zmvfgP9NX1UZu6PKchm47fQG3PDuajJyCnjnmpb8sWwHa7fuLSqzKSuf/3trFTv3eDi+ZQx3n9GQq0ankl9oGfnOKnbne3G7YPTwFkxfsZPFaXn899N1RcvfMLgeu/Z4AWiaEsnADvFc/OJykmPCefHK5pz//DKMgZuH1OeiF5ezI8/DyJPrcV73ZN78fQvL03dzxWsr2FtgObtrEiMH1S9Rv5SPy8B1w1px95N/sW37Xl4c1YUZ87axftP+Dytbtu7m9kfnsSuvkC4dE7nxyjbc+OAcmjSoySl96nHDA3MoKLQ8eltHZs7PZNOW3cxdvJ23P1uN12u56vzmXHhaY976dDU7dhZw33ML2J6dT5MGNXn09k5cctM0AB55eTF5ezwA3DuyHSd2q82kmRm8/lFqUVvOGNCAFk1igruTqgCvx8PHz7/KDc88TEJKMo9ffTMde/agXtP919zFM2eTkbaJB8a8wZolyxn77Mvc+dpzZS47fsxntOnciUGXnM/4MZ/y85jPOOuaf5G+dj2zJ0zm3ndfZUdmJi/c8h8e+HA0Lrebsc++wiW3XU+zdm146Y77WTxzDu17dGHauPFEx9TiwY/eZNZvk/jq9XcYPuquEO61f6Yj5dbbkRHnKkGzhJpk7NrLtrx8PNYyKy2LTnXjSpTZmV/Iuuw8PNYesPxej++N0e3y9Tbss3nXHrbs2ntA+U5145iVlkWh15KZl0/Grr00S6gJwJqsPHbsLTxgmT2F3qK/I9zV9lD9LUc3jCYtM59NWfkUeiy/LMymd9uSx3nhhjx2+t/UFm3IIyUuomje7nzfMQhzG8LchkD6t4/nlwVZAPRuG8cvC7N9vQ/Z+aRl5nN0w2gAjDFEhfuOY3Ski607fb1Gc9fksrfA9xpblJZH7djww7X51Urr5rFs2rKbzVv3UOixTJy5heOPSy5RZklqDrvyfOfastQckhN9vYKN60ezdFUOe/O9eL2WBcuy6dk5BYC5i7Lwen3HZ+mqHJITfMusWr+L7dn5AKzbmEtEuIvwMN9rZF9IcrsNYWEuOPASQt8edZg4Y8uBM6RMa5euIKVBfVLq1yMsPJwu/Xozf8qMEmXmT5lBj0H9MMbQvF0b8nblsiNze5nLzp86gx6DBwDQY/AA/to3fcoMuvTrTXhEOMn16pLSoD5rl65gR+Z29uTl0bx9W4wx9BjUj/lTpvvrmkmPQf0BOO6kXiybOx8b4H2kuvO6XRV6hEq17VGKjwpn++78oufZewpolhBd7uUN8N++bUipGcnE1VtZk5VXZvmEqHBWb99fJmtPPvFR4ZBV9nr6NEtmYIs6uI3h2akry90+8UmJDSdjx/7jnLGjgHYNnY/z6Z0TmbEip+i5y8C7/9eKhokRfPFnJovTSh7nY5rUZPuuQjZs960jJSacxWm5+9eXk09KbDiLNsCT36UxZmRrdhd42ZC5l6e/33jg+o9LZPrKnAOmy8ElJUSydfv+HuFt2/fS5qhYx/KDT6rHrAWZAKxNy+WKc5sTUzOM/AIvXTslsXLNzgOWGXRiPSb9mXHA9F5dUli1bicFhfvfDB+5rROtm8cwe8F2/phVcpnaSZHUSanBX0sOcgGQA2RvyySh9v4AnJCSzJqlywOUSSlRJntrZpnL7szKJi4pEYC4pER2ZmUX1dXs6NbFlkkie1sm7rAw4lOSiqbHpySTvS3zgPW7w9xE1Ywmd0cOteJLfkir7o6UW29HRisrQaC+gYrkfQs89Psy7hy/iGYJNakfc7D/BTlwb8TBTFyzjf/8spgvl2xkSOu6h1RHdVaRvX5cs5qc0TmRl35OL5rmtTDslRWc8fQSjm4QTfPaJY/zyR3j+WVB9v71Obyw3C44u2sSw15ZwWlPLiF18x4u7127RLHBneJp2yCKD6dsrUCrZZ9A+97pQ3ynNvEM6l2Ptz7x3crekJ7Hp+PW89gdx/DIbZ1Ys34XHm/JhS86vQker2XCtJK9QE0aRHPVBUfxwrsl36z/8/R8LrpxGuFhhmOOTigxr0/3OkyZtRWvOhkqLFDPzAGHPtCBN+VcthzrwxiHukwZ6z+094CqzLpcFXqESrUNSlm7C0iM2n+LJb5GONm7Kz6AdneBh+XbdtKujvMnV9/68kmI2n9LJaFGRIXWNyst64CxUnJwGTkF1C52K612XHjRLa/iWtSpwT1nNuL2MWvI2e05YP6uPV7mrt1Fj5b7x5S4XdDn6Dh+WZTtvL7YCLbuLKBVvSgANmb5ep5+W5RNh8Y1i8p1bV6LK06qw+1j1lLg0bvnodi2fS8pifuDbHJiJJnZB94Gb9aoJjdd1YZRzy9kZ+7+W97jJ6cz8v7Z3PboPHbmFrJx8/7ewwE969LtmCSeeG1JibqSEyK574YOPDV6KekZB37RoqDAy/R5mQfcAjypR23ddjtECSnJZGVsK3qetXUbcclJJcrEpySTlbG1RJn45KQyl41JiGdH5nYAdmRuJyYh3mF9mcQnJZJQ29dLtU/21m3EJScesH5PoYfduXnUjNV4tNK8LleFHqFSbYPS2uxcateKJCk6ArcxdG2YwPzNO8q1bK2IMKLCfYPQwl2GtimxbD7It9Hmb95B14YJhLkMSdER1K4VyZqs3DKXqV0zsujvDnXj2LJL33irqKUb82iUFEG9+AjC3IaBHeL5Y1nJ41wnLpzHLmrKA5+vZ0Pm/tt08dFuatXwnSKRYYauzWuxbuv+Y9C1eQxrt+5la87+4PXHsh0M7BBPuNtQLz6CRkkRLEnLY2tOAc1q1yA+2ve66dYihrX+ulrVi+LOoQ25/cM1ZOUeOFZNymf5mp00qBNFneQahLkNfbrXYca8bSXKpCRGct/17Xnq9SVs3LK7xLy4mPCiMj07JzNxhu92WZcOiZx/amNGPb+Qvfn7xw3WjA7joVs68s5nq1mycv9rqkakm0R/WHa5DN06JbIhfX/oalg3ilrRYSxJ1S3WQ9GkTSsy0jayLX0zhQUFzJ4wmY49u5co07Fnd2aMn4C1ltWLlxFVsyZxSYllLtuxZ3dm/PQrADN++pVOPXsUTZ89YTIF+QVsS99MRtpGmrZtRVxSIjWioli9eBnWWmaMn0CnXj2Krf83AOZOmkLrYzti1KN0AOt2VegRKtV2jJLXwtgFG7jphBa4jGHqukzSd+6hd1PfJ7/Ja7cRGxnGf/q0oUaYG4tlwFG1uf+3JcTVCOfK45rgMgZjYPbGLBZu8V30jqkXx0UdG1ErIozrexzFhh27eWF6Kuk79zBnYzYP9D8aj9cydv6Golt957RrQLeGCUS4XTwxqD1T1mXy3bJ0+jZPoW1KDB5rycv38M5cfROqojxeePr7jbxweXNcLvh+7nbWZOzlrK6+T5Ffzcrkqj51iIt2c/vpDf3LWK58bSXJMeHce05j3MbXa/7boh1MXbF/3MrADvH8sjC7xPrWZOzlt0XZjL2hNR6v5envN+K1sG1nIW/9voXXhreg0GPZnJ3Pg19uAOD6QfWIjnDxyIVNAdiyI5/bx6yt9H1T1Xi9lpc/WMGjt3fC5TL8PDmddRvzOLVvfQDG/b6JS85sSkytcEYOawX4jvX1o+YAcN/17YmpFY7HY3npg5VFg76vu6wl4WEuHru9EwDLVuXwv/dWcMaABtSvE8XFZzTh4jOaAHD3U/MxBkbd1IHwcBdul+GvJVl8P2FTUTv79KjDpJkHjnOS8nGHubnwpv/jxdvuxev1csKQgdRv1oTJ3/wAQO+hQ2jfoyuLZszmvouH+34e4K6by1wWYNDF5/HmqMeZOu4XEuuk8O8H7gagfrMmdO7biwcvvwaX282FN12Ly+37wHPRLdfx3uPPUbB3L+26d6Fd9y4A9BxyMu8+8jT3XTyc6JgYrrr/jmDvpiOCdR0Z4dEEYSS+HfH13Mpeh4TY6DOPo8e98w9eUI5oMx7yhYVBl/8e4pZIZRr/Xl8AJmxOPUhJOdL1q9siZGml1y1zKxRApjx7XEjaWm17lERERCR0jpQeJQUlERERCToFJREREREH9sj4YW4FJREREQk+G6YeJREREZHA1KMkIiIi4uAI+SVHBSUREREJPgUlERERkcCMgpKIiIhIYMZ1ZPy/lkdInhMREZGqxLgq9ihXncYMNsYsN8akGmPuCjD/EmPMAv9jmjGm08HqVI+SiIiIBJ3rMH/rzRjjBl4GBgJpwCxjzLfW2iXFiq0BTrLWZhljTgFGA90PrG0/BaUgmfXhSwB0vXRkiFsilSlj7ucA1D7u3BC3RETkn811+O9pdQNSrbWrAYwxHwNDgaKgZK2dVqz8DKDhwSpVUAqCGe88y6LvPwbAU1hAjytuDnGLpLLk79wa6iaIiBwRKjpGyRgzAhhRbNJoa+3oYs8bABuKPU+j7N6iq4AfD7ZeBaVKtnbm72xe8hftTr0AgM2L57F25kSadu8T2oaJyCGx1mLMkfGLwnLodJwrX0V7lPyhaHQZRQIdsIBpzBjTF19Q6nWw9WowdyVr2r0vpz70OhHRtYiIrsWpD72ukCRyhLJeDwv/eJgt6yaHuilSiTweD49cezN/jBsf6qZUaS5XxR7lkAY0Kva8IbCpdCFjTEfgTWCotTbzYJWqRykIwmtEBfxbRI4c1uth4ZRHKSzII6Vhj1A3RyqJx+Ph0etuZfeuXXQf0CfUzanSKmGM0iygpTGmGbARuBC4uHgBY0xj4EvgMmvtivJUqqAkIlIOm9dOZPPa3wH4dczggGXanXAHDVoEnidHhonfjGPiN+MAOKVp+4Blbn/+cQZfcE4wm1UlHe6gZK0tNMaMBMbj+5/k3rbWLjbGXOOf/xpwH5AEvOK/tVpore1SVr0KSiKHQfaqqWQu+pHIuPoArPrmvyS1P4X4o3qGuGVyuNRt2oeMDVPxFO7hmD4P4HKHh7pJUgn6DD2VaeN/Y3deHg+89TLhERGhblKVVQk9SlhrfwB+KDXttWJ/DweGV6ROjVESOQxim3ShIDeTzbPHsnn2WApyM4ltUuaHFDnCGJebjif+h7DwKLamTQ91c6SSuN1u7nnlWaJq1mT6zxNC3Zwqze2yFXqEinqURA4DV1gkrc5/nuWf3AhAq/OfxxUWGeJWyeFmXG46nPhffRuqinO73fz31ed0nCtZZfQoVQYFpSDpfOHVoW6CVDJXWCRtL3nt4AXliKY3z+pBx7nyKSiJiIiIOHArKImIiIgE5jpCOu0UlERERCTo1KMkIiIi4iD8CEkgR0gzRUREpCpRj5KIiIiIAwUlEREREQf6eQARERERB259601EREQkMN16ExEREXGgoCQiIiLiIOwI+cVJBSUREREJOvUoiYiIiDjQYG4RERERB+pREhEREXGgoCQiIiLiwK3B3CIiIiKBqUdJRERExIEGc4uIiIg40K03EREREQe69SYiIiLiQD1KIiIiIg7UoyQiIiLiwGXUoyQiIiISkHqURERERBxojJKIiIiIg4gwBSURERGRgNSjJCIiIuJAY5REREREHLjUoyQiIiISmG69iYiIiDjQrTcRERERB0dKj9IRkudERESkKnG5TIUe5WGMGWyMWW6MSTXG3BVgvjHG/M8/f4Ex5riD1akeJREREQm6w33rzRjjBl4GBgJpwCxjzLfW2iXFip0CtPQ/ugOv+v91pKAkIiIiQVcJt966AanW2tUAxpiPgaFA8aA0FHjfWmuBGcaYeGNMPWttulOlxle2UlX6CkREROSQHBkDhQBjzAhgRLFJo621o4vNPxcYbK0d7n9+GdDdWjuyWJnvgcettVP8z38D7rTWznZab1B6lCZsTg3GaiSE+tVtwYB+o0LdDKlkv04YBUDvo9qGtiFSqSavWgro2l0d9KvbItRNKDd/KBpdRpFAoa90Z015ypSgwdwiIiJSFaQBjYo9bwhsOoQyJSgoiYiISFUwC2hpjGlmjIkALgS+LVXmW2CY/9tvPYAdZY1PAg3mFhERkSrAWltojBkJjAfcwNvW2sXGmGv8818DfgCGAKlAHnDlwepVUBIREZEqwVr7A74wVHzaa8X+tsB1FalTt95EREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxoKAkIiIi4kBBSURERMSBgpKIiIiIAwUlEREREQcKSiIiIiIOFJREREREHCgoiYiIiDhQUBIRERFxEBbqBoTS4pmz+fTF0Vivl56nnsygS84vMd9ay6f/e53FM2cTERnJsLtvpnGrFmUum5uzkzdHPU7m5gyS6tZm+AN3UTMmBoCfPvyUaT/8jHG5uOCGqzm6W2cA1i1fyfuPPUdBfj7tunfh/BuuxhhDQX4B7z36DOtXpFIzNobh999FUr06QdxDVUPXri24duRgXC4XP/4wl4/HTikx/4QTWnPFlf3wWovH4+XVl39i0aL1NGyUxH/vPa+oXL16Cbz37u98+cUMep90NMMu70PjximMvPYNVqzYBIDb7eLW286gZct6uNwufv15PmPHTiEqKoLnXvhXUV0pKbH8+usCXn35J8LD3dx511m0bFWfnJw8Hn7wc7ZsyQ7KvqlquvXuxQ333oPL7WLcJ58z5vU3DyhzTPeuXP/fuwkLC2dHVhY3XDyMiIgIXvz4A8IjInC7w5j403jeeeGlEstdOPxKrr37Dk7vcjw7srKp26A+H/w8jvWr1wCw5K/5PHPvAwD0O/UULrv2alwuN9MnTuK1J54GYPA5Z3LtnbezdcsWAL784CPGffp5Ze6SKknXbgmmatuj5PV4+Pj5Vxn55APc996rzPptMulr15cos3jmbDLSNvHAmDe4+LbrGfvsywdddvyYz2jTuRMPfvQGbTp34ucxnwGQvnY9sydM5t53X+X6px5k7HOv4PV4ABj77Ctcctv1PDDmDTLSNrF45hwApo0bT3RMLR786E36nXcmX73+TrB2T5Xhchmuv3EI99w1hquufJm+/drTuElKiTJz565hxL9f5ZoRr/H0U99wy21nAJC2IZNrRrzGNSNe49prXmfv3gKmTFkKwNo1GYy6/xMWLlhXoq6TTmpHeHgY/x7+KtdeM5pTT+9CnTrx7N6dX1TXNSNeY8uWbKb84avrlFOOY+fOPVx+2f/44vMZ/HvEgCDsmarH5XJx86h7uf1fIxg26HT6n34qTVocVaJMrZgYbnngPu4ecR2Xn3I6911/EwD5+fncdOmV/Ou0s/jX6WfRvXcvjj6mU9FytevVpUvPE9i8cVOJ+jau38BVp5/NVaefXRSSYuPj+b+7buOmy67k8lNOJzE5ieNO6FG0zIRxPxYto5BUcbp2S7BV26C0dukKUhrUJ6V+PcLCw+nSrzfzp8woUWb+lBn0GNQPYwzN27Uhb1cuOzK3l7ns/Kkz6DHY90bXY/AA/to3fcoMuvTrTXhEOMn16pLSoD5rl65gR+Z29uTl0bx9W4wx9BjUj/lTpvvrmkmPQf0BOO6kXiybOx9rbbB2UZXQuk0DNm3cTnp6FoWFHiZOWETPE1qXKLNnT37R3zVqhAfcx8ce15xNm7aTsWUHAOvXbyNtQ+YB5SyWGlHhuFwuIiPDKCzwkJe3t0SZBg0SiY+vWRSyTujZmp9//guAyZOWcOxxzf/WNldXbTt1ZOO69aRvSKOwoIDfvv+BXgP6lSgz4IzTmPzzr2SkpwOQnbm9aN7uvDwAwsLCCAsr+ToY+Z+7ePWJp8t1/tVv1JANa9axY3sWALOnTuekQSf/7e0TH127JdiqbVDK3pZJQu3koucJKclkb8sMUCalZJmtmWUuuzMrm7ikRADikhLZmZXtsL4ksrdlkr01k/iUpKLp8cXqKr5+d5ibqJrR5O7IORybX20kJ8eSkbF/n23dlkNSSuwB5Xr2asPb747kkUcv4emnvjlgft++7fl9wqKDrm/ypCXs2V3Ap5/fypixN/PZp9PYuXN3ybr6dWDixMVFz5OSY9nqb6PX6yU3dw+xsdHl3kbxSa5Tm4z0zUXPt27eQkqdkrc7GjVrSkxsLC+MeY83vvmcQWcNLZrncrl467sv+ebPKcyeOo2l8xcA0LN/X7Zt2cKqZcsPWGe9hg1489sv+N9H79Oxi+92TNq69TRu3oy6Derjdrs5cWB/aterW7TMSYNP5p1xX/PgS8+XmC7lo2u3BFu1HaMUKN2bAwsduKAp57LlWB/GONRlylj/wdYkxQXcXQH269Qpy5g6ZRkdOjbhyiv7ccft7xfNCwtzc/wJrXnzzV8Pur42bRrg9Xq54LxniImJ4rkXrmTu3NWkp2cVlenbtz2PP/Zl2W1Enz4rygTYkbbUfnS73bRq346bL7uSyBqRvPr5xyyeN5+0tWvxer1cdfrZ1IqJ4eHXXqRZq5ZsWr+By669mlsvH35A3Zlbt3Leif3Jyc6mVfujefS1lxg2+HR25eTw7H0PMOp/z+L1WhbPnUe9xo0AmPbbRH77bhwF+QWccdEF3PPUY9x06ZWVs0OqKF27JdiqbY9SQkoyWRnbip5nbd1GXHJSiTLxKclkZWwtUSY+OanMZWMS4tnh787fkbmdmIR4h/VlEp+USEJt3yedfbK3biMuOfGA9XsKPezOzaNmbMzh2PxqY+vWHGrX3t+DlJIcS+a2nY7lFy5YR736CSV6dLp1a8HKlelkZ+UedH39+ndg1qxUPB4v2dm5LF60gVat6hfNb968Dm63i5Ur04umbduaQ4q/jS6Xi5o1a5CTs/uAuqVsWzdvKdFDk1K3Dtu2ZJQqs5k/J//Bnt272ZGVzfw/Z9Oibclbsbt27uSvGX/SvXcvGjRuRL1GDXl73Nd8MulXUurW4c1vvyAxOZmC/AJysrMBWLFoCRvXbaBRs6YATJswkWvOuZBrz7uI9WvWkLbWd5s1JzubgvwCAL7/5DNatW9XSXuj6tK1W4Kt2galJm1akZG2kW3pmyksKGD2hMl07Nm9RJmOPbszY/wErLWsXryMqJo1iUtKLHPZjj27M+MnX8/DjJ9+pVPPHkXTZ0+YTEF+AdvSN5ORtpGmbVsRl5RIjagoVi9ehrWWGeMn0KlXj2Lr/w2AuZOm0PrYjgE/NYuz5cs20aBBEnXrxhMW5qZPv/ZMm17yFkr9+olFf7doWY/wcDc5OXlF0/r268DvExaWa30ZGTs45thmgG+8U9u2DVm/Yf9Ftl//DkwoVde0acs5+eRjAOh90tH8NW9NhbZRfJYtWEjDpk2o17ABYeHh9D9tCFN/+71EmSm/TqBj18643W4ia9Sg7TEdWbdqNXGJCdTyf8MpIjKSzj2PZ92qNaxesZKh3XpxwUkDuOCkAWzdvIXhZ5zD9m3biEtMwOXyXULrNWpIw6ZN2LQ+DYB4/y2cWrGxnHnJRXz/iW/QdlLK/ttBPQf0Y13q6krfL1WNrt0SbNX21ps7zM2FN/0fL952L16vlxOGDKR+syZM/uYHAHoPHUL7Hl1ZNGM291083PcV07tuLnNZgEEXn8ebox5n6rhfSKyTwr8fuBuA+s2a0LlvLx68/BpcbjcX3nQtLrcbgItuuY73Hn+Ogr17ade9C+26dwGg55CTefeRp7nv4uFEx8Rw1f13BHs3HfG8Xi8vvvgDjz9xGS634acf57Fu7VZOO923j7//bjYn9m7LwJM7UVjoJX9vAQ8/uP+bSJGR4XTu3Jznn/uuRL09e7Vh5PVDiIuL5pFHL2bVqs3cdeeHfPP1LG6/cyhvvn0tBsP48fNYs3pL0XInndSOe+4eU6KuH3+Yx133nMV7H9zAzp27eeQhfRPqUHg8Hp5/4GGefvdNXC4XP3z+JWtXpnLGRRcA8O3YT1i3ajUzJ0/hnXFf47WWcZ98zpoVK2neuhX3PPUYbrcb43Lx+7ifmP77xDLXd0zXLvzrphvweArxerw8c+8odu7wDfa/4d57aNHG11P17kuvkrZ2LQDnXH4pPfv3w+MpJGfHDh674+5K2x9Vla7dEmwmCCPx7YTNqZW9DgmxfnVbMKDfqFA3QyrZrxNGAdD7qLahbYhUqsmrfD9doWt31devbgt1dR1Etb31JiIiInIwCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASEamg9Jxs0nOyQ90MEQmCsFA3QETkSJNfWBDqJohIkKhHqZJZa0PdBAkRHXuRI5fOX9lHQakSeTweHrn2Zv4YNz7UTZEgy8pex/wFH+HxFIa6KSJSQbp2S3EKSpXE4/Hw6HW3krdzF90H9Al1cyTI4uMaEx5RkwULxyosiRxBdO2W0jRGqZJM/GYcE78ZB8ApTdsHLHP7848z+IJzgtksOcwyM1P5a8GYMsts3DSLxo2OD1KLROTv0LVbSlNQqiR9hp7KtPG/sTsvjwfeepnwiIhQN0kqQVJSC/r3vf+A6dZaliz7hvy9O2lQv0sIWiaVYXveLrbuyiEyLByA5RmbSKkVS2J0rRC3TA4XXbulNN16qyRut5t7XnmWqJo1mf7zhFA3R4Ise8d68vN30bHDhbjd4aFujhwm8TWiyfd4SM/JIj0ni3yPh/ga0aFulhxGunZLaepRqkRut5v/vvocxphQN0WCLCG+CfFxjXXsqxiXy0W7Og1YvCUNgHZ1GuBy6fNmVaNrtxSnoFTJdKJVXzr2VZPL5aJDvcahboZUMp2/so8+ComIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcaCgJCIiIuJAQUlERETEgYKSiIiIiAMFJREREREHCkoiIiIiDhSURERERBwoKImIiIg4UFASERERcWCstZW9jkpfgYiIiBwSE+oG/NOFBWEd1e4gGGNGWGtHh7odIiIi8vfo1lvlGBHqBoiIiMjfp6AkIiIi4kBBSURERMSBglLl0PgkERGRKiAY33oTEREROSKpR0lERETEgYLSYWaMGWyMWW6MSTXG3BXq9oiIiMih0623w8gY4wZWAAOBNGAWcJG1dklIGyYiIiKHRD1Kh1c3INVau9pamw98DAwNcZtERETkECkoHV4NgA3Fnqf5p4mIiMgRSEHp8Ar037Xo3qaIiMgRSkHp8EoDGhV73hDYFKK2iIiIyN+koHR4zQJaGmOaGWMigAuBb0PcJhERETlEYaFuQFVirS00xowExgNu4G1r7eIQN0tEREQOkX4eQERERMSBbr2JiIiIOFBQEhEREXGgoCQiIiLiQEFJRERExIGCkoiIiIgDBSURERERBwpKIiIiIg4UlEREREQc/D9EGtuR2PU0sQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 792x648 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from numpy import linalg\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "import seaborn\n",
    "from matplotlib.colors import LogNorm\n",
    "env = gym.make('FrozenLake-v0')\n",
    "\n",
    "# print('Number of Actions', env.nA) \n",
    "# print('Number of States ', env.nS)\n",
    "# print('P[10,3]', env.P[10][3])\n",
    "\n",
    "def fancy_visual(value_func,policy_int):    \n",
    "    grid = 4    \n",
    "    f, ax = plt.subplots(figsize=(11, 9))\n",
    "    cmap = seaborn.diverging_palette(220, 10, as_cmap=True)\n",
    "    reshaped = np.reshape(value_func,(grid,grid))\n",
    "    seaborn.heatmap(reshaped, cmap=\"icefire\",vmax=1.1, robust = True,\n",
    "                square=True, xticklabels=grid+1, yticklabels=grid+1,\n",
    "                linewidths=.5, cbar_kws={\"shrink\": .5}, ax=ax, annot=True, fmt=\"f\")\n",
    "    counter = 0\n",
    "    for j in range(0, 4):\n",
    "        for i in range(0, 4):\n",
    "            if int(policy_int[counter]) == 1:\n",
    "                plt.text(i+0.5, j+0.7, u'\\u2193', fontsize=12)\n",
    "            elif int(policy_int[counter]) == 3:\n",
    "                plt.text(i+0.5, j+0.7, u'\\u2191', fontsize=12)\n",
    "            elif int(policy_int[counter]) == 0:\n",
    "                plt.text(i+0.5, j+0.7, u'\\u2190', fontsize=12)\n",
    "            else:\n",
    "                plt.text(i+0.5, j+0.7, u'\\u2192', fontsize=12)\n",
    "            counter=counter+1\n",
    "\n",
    "    plt.title('Heatmap of policy iteration with value function values and directions')\n",
    "    print('Value Function \\n',value_func)\n",
    "    print('Policy \\n',policy_int)\n",
    "    plt.show()\n",
    "    \n",
    "def value_iteration():\n",
    "    V = np.zeros(env.nS)\n",
    "    V_ = np.zeros(env.nS)\n",
    "    diff = []\n",
    "    for t in range(100):        \n",
    "        for x in range(env.nS):            \n",
    "            Q = []\n",
    "            for a in range(env.nA):\n",
    "                q = 0\n",
    "                for i in range(len(env.P[x][a])):\n",
    "                    p, s_, r, done = env.P[x][a][i]\n",
    "                    q += p * (r + gamma * V[s_])\n",
    "                Q.append(q)\n",
    "                a_max = np.argmax(np.asarray(Q))\n",
    "                V_[x] = Q[a_max]\n",
    "        diff.append(linalg.norm(V_ - V, 2))\n",
    "        if diff[-1] < epsilon:\n",
    "            print(\"Convergence Plot of Value Iteration - \")\n",
    "#             print(\"Number of iterations to converge: \", t+1)\n",
    "            plt.plot(range(int(t+1)), diff)\n",
    "            plt.show()\n",
    "            break\n",
    "        else:\n",
    "            V = V_.copy()   \n",
    "    return V\n",
    "\n",
    "def optimal_policy(V):\n",
    "    pi = np.zeros(env.nS, dtype=int)\n",
    "    Q_star = np.zeros((env.nS, env.nA))\n",
    "#     pi = np.zeros((env.nS, env.nA))\n",
    "    for x in range(env.nS):\n",
    "        Q = []\n",
    "        for a in range(env.nA):\n",
    "            q = 0\n",
    "            for i in range(len(env.P[x][a])):\n",
    "                p, s_, r, done = env.P[x][a][i]\n",
    "                q += p * (r + gamma * V[s_])\n",
    "            Q.append(q)\n",
    "#         Q_star.append(Q)\n",
    "        Q_star[x, :] = Q\n",
    "        a_star = np.argmax(np.asarray(Q))\n",
    "        pi[x] = a_star\n",
    "    return pi, Q_star\n",
    "\n",
    "gamma = 0.9\n",
    "epsilon = 5e-3\n",
    "V_star = value_iteration()\n",
    "pi_star, Q_star = optimal_policy(V_star)\n",
    "fancy_visual(V_star, pi_star)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "other-expression",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimal Q-Value Function - \n",
      " [[0.05290655 0.05162303 0.05162303 0.04437451]\n",
      " [0.02915516 0.03425769 0.03297416 0.04819351]\n",
      " [0.0647469  0.05901315 0.06411567 0.04627875]\n",
      " [0.03234294 0.03234294 0.02660918 0.04564753]\n",
      " [0.07699103 0.06177169 0.05452317 0.03768721]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.10670947 0.08767113 0.10670947 0.01903834]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.06177169 0.11042842 0.09359246 0.13289629]\n",
      " [0.15086494 0.23853606 0.19923224 0.12697495]\n",
      " [0.29327591 0.26150317 0.22215132 0.10289733]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.        ]\n",
      " [0.18268571 0.30193969 0.37306429 0.26150317]\n",
      " [0.38961082 0.63527302 0.61138304 0.53256557]\n",
      " [0.         0.         0.         0.        ]] \n",
      "\n",
      "Optimal State Value Function - \n",
      " [0.05073115 0.04645272 0.06346115 0.04434864 0.07489288 0.\n",
      " 0.10590912 0.         0.13101275 0.23708198 0.29223709 0.\n",
      " 0.         0.37187037 0.63459526 0.        ] \n",
      "\n",
      "Optimal Policy - \n",
      " [0 3 0 3 0 0 0 0 3 1 0 0 0 2 1 0] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Optimal Q-Value Function - \\n\", Q_star, \"\\n\")\n",
    "print(\"Optimal State Value Function - \\n\", V_star, \"\\n\")\n",
    "print(\"Optimal Policy - \\n\", pi_star, \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "identified-biography",
   "metadata": {},
   "outputs": [],
   "source": [
    "## End of Part 4."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}