Part2-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d133a72",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-02-11 15:50:32.417625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import torch\n",
    "import torchvision as tv\n",
    "from torchvision import transforms, datasets\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1ff30450",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create random data between (-10, 10) and determine groundtruth\n",
    "simulatedInput = 20 * torch.rand((1000, 1)) - 10\n",
    "groundTruth = np.cos(simulatedInput)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c9eca80d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the number of parameters in a neural network\n",
    "def calcParams(inputModel):\n",
    "    val = sum(params.numel() for params in inputModel.parameters() if params.requires_grad)\n",
    "    return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1f0f1633",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up NN for cos(x) training - 2 Hidden Layers, 32 parameters - Shallow Network\n",
    "class OptimizeNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.fc1 = nn.Linear(1, 3)\n",
    "        self.fc2 = nn.Linear(3, 5)\n",
    "        self.fc3 = nn.Linear(5, 1)\n",
    "\n",
    "    def forward(self, val):\n",
    "        val = F.relu(self.fc1(val))\n",
    "        val = F.relu(self.fc2(val))\n",
    "        val = self.fc3(val)\n",
    "        return val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a7e0846b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up necessary auxilaries for neural net training\n",
    "optimNet = OptimizeNN()\n",
    "costFunc = nn.MSELoss()\n",
    "opt = optim.Adam(optimNet.parameters(), lr=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a3bdac8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def calculate_grad_norm(model, criterion, train, target):\n",
    "#     model.train()\n",
    "#     model.zero_grad()\n",
    "#     output = model(train)\n",
    "#     loss = criterion(output, target)\n",
    "#     loss.backward()\n",
    "\n",
    "#     grads = []\n",
    "#     for p in model.modules():\n",
    "#         if isinstance(p, nn.Linear):\n",
    "#             param_norm = p.weight.grad.norm(2).item()\n",
    "#             grads.append(param_norm)\n",
    "\n",
    "#     grad_mean = np.mean(grads) # compute mean of gradient norms\n",
    "#     return grad_mean,loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "56b8801e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def min_ratio(w):\n",
    "    count = 0\n",
    "    total = 0\n",
    "    for x in w:\n",
    "        if(x>0):\n",
    "            count += 1\n",
    "        total +=1\n",
    "    return count/ total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5f6a2ff7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_grad_norm(model):\n",
    "#     model.train()\n",
    "#     model.zero_grad()\n",
    "#     output = model(train)\n",
    "#     loss = criterion(output, target)\n",
    "#     loss.backward()\n",
    "\n",
    "    grads = []\n",
    "    for p in model.modules():\n",
    "        if isinstance(p, nn.Linear):\n",
    "            param_norm = p.weight.grad.norm(2).item()\n",
    "            grads.append(param_norm)\n",
    "\n",
    "    grad_mean = np.mean(grads) # compute mean of gradient norms\n",
    "    return grad_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "75e66cc0",
   "metadata": {},
   "outputs": [],
   "source": [
    "minRatio = []\n",
    "Loss = []\n",
    "train_count = 100\n",
    "for i in range(train_count):\n",
    "    EPOCHS = 30\n",
    "    lss = 0\n",
    "    # Set up necessary auxilaries for neural net training\n",
    "    optimNet = OptimizeNN()\n",
    "    costFunc = nn.MSELoss()\n",
    "    opt = optim.Adam(optimNet.parameters(), lr=0.01)\n",
    "    for epochIndex in range(EPOCHS):\n",
    "        optimNet.zero_grad()\n",
    "        output = optimNet(simulatedInput)\n",
    "        cost = costFunc(output, groundTruth)\n",
    "        with torch.no_grad():\n",
    "            lss += (costFunc(output, groundTruth))\n",
    "        cost.backward()\n",
    "        opt.step()\n",
    "    #print(calculate_grad_norm(optimNet))\n",
    "    Loss.append(lss/len(simulatedInput))\n",
    "    num_param = calcParams(optimNet)\n",
    "    \n",
    "    # Allocate Hessian size\n",
    "    H = torch.zeros((num_param, num_param))\n",
    "\n",
    "    y_hat = optimNet(simulatedInput)\n",
    "    y = simulatedInput\n",
    "    loss  = ((y_hat - y)**2).mean()\n",
    "    # Calculate Jacobian w.r.t. model parameters\n",
    "    J = torch.autograd.grad(loss, list(optimNet.parameters()), create_graph=True)\n",
    "    J = torch.cat([e.flatten() for e in J]) # flatten\n",
    "\n",
    "    # Fill in Hessian\n",
    "    num_param = calcParams(optimNet)\n",
    "    for i in range(num_param):\n",
    "        result = torch.autograd.grad(J[i], list(optimNet.parameters()), retain_graph=True)\n",
    "        H[i] = torch.cat([r.flatten() for r in result]) # flatten\n",
    "    w, v = np.linalg.eig(H)\n",
    "    minRatio.append(min_ratio(w))\n",
    "    #print(min_ratio(w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2866e56c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#costFunc1 = calculate_grad_norm(optimNet, costFunc, simulatedInput, groundTruth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "44c105e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# EPOCHS = 30\n",
    "# for epochIndex in range(EPOCHS):\n",
    "#     optimNet.zero_grad()\n",
    "#     output = optimNet(simulatedInput)\n",
    "#     cost =  torch.from_numpy(calculate_grad_norm(optimNet))\n",
    "#     cost.backward()\n",
    "#     opt.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "715a437c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0317182457074523"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calculate_grad_norm(optimNet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "148e82a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_param = calcParams(optimNet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "71c49b76",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Allocate Hessian size\n",
    "H = torch.zeros((num_param, num_param))\n",
    "\n",
    "y_hat = optimNet(simulatedInput)\n",
    "y = simulatedInput\n",
    "loss  = ((y_hat - y)**2).mean()\n",
    "# Calculate Jacobian w.r.t. model parameters\n",
    "J = torch.autograd.grad(loss, list(optimNet.parameters()), create_graph=True)\n",
    "J = torch.cat([e.flatten() for e in J]) # flatten\n",
    "\n",
    "# Fill in Hessian\n",
    "for i in range(num_param):\n",
    "    result = torch.autograd.grad(J[i], list(optimNet.parameters()), retain_graph=True)\n",
    "    H[i] = torch.cat([r.flatten() for r in result]) # flatten"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e3ffb963",
   "metadata": {},
   "outputs": [],
   "source": [
    "# H.numpy().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ec0fac40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# w, v = np.linalg.eig(H)\n",
    "# w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e536147d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# min_ratio(w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ec5ac242",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlIAAAGwCAYAAABiu4tnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8o6BhiAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNoUlEQVR4nO3de1xUdf4/8NdwHTUgBGFAEQfN1LAUKMUiXVMU/ZYXLNNvJVluZG0B6y9vtaK2spqVa4imiZu5mvtdrWzlq2AJqzHmDcwLW+aikM4sgQFeQeH8/uA7kyMDzBwZPmeY1/PxmEfNmffM+cwcxvOe9+dyVJIkSSAiIiIim7mIbgARERGRo2IiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSyU10A9qz+vp6XLhwAV5eXlCpVKKbQ0RERFaQJAmXLl1CcHAwXFyarzkxkbKjCxcuICQkRHQziIiISIbS0lJ069at2RgmUnbk5eUFoOFAeHt7C24NERERWaO6uhohISGm83hzmEjZkbE7z9vbm4kUERGRg7FmWI7wweYZGRnQarVQq9WIjIzEvn37mo3Py8tDZGQk1Go1wsLCsGbNGrPHt2/fjqioKNx9993o1KkTBgwYgE8++cQsJi0tDQ8++CC8vLwQEBCA8ePH4/vvvzeLSUhIgEqlMrsNHjy4dd40ERERtQtCE6mtW7ciKSkJ8+fPR0FBAWJiYhAXF4eSkhKL8cXFxRgzZgxiYmJQUFCAefPm4bXXXsO2bdtMMZ07d8b8+fOh0+nw3Xff4fnnn8fzzz+P3bt3m2Ly8vLwyiuv4MCBA8jJycHNmzcRGxuLK1eumO1v9OjR0Ov1pltWVpZ9PggiIiJySCpJkiRROx80aBAiIiKwevVq07a+ffti/PjxSEtLaxQ/e/Zs7NixA0VFRaZtiYmJOHbsGHQ6XZP7iYiIwNixY7F48WKLj//8888ICAhAXl4eHn30UQANFanKykp8/vnnVr+fmpoa1NTUmO4b+1irqqrYtUdEROQgqqur4ePjY9X5W1hFqra2FkeOHEFsbKzZ9tjYWOTn51t8jk6naxQ/atQoHD58GDdu3GgUL0kSvvrqK3z//femBMmSqqoqAA3VrFvl5uYiICAAvXv3xowZM1BWVtbse0pLS4OPj4/pxhl7RERE7ZuwRKq8vBx1dXUIDAw02x4YGAiDwWDxOQaDwWL8zZs3UV5ebtpWVVWFu+66Cx4eHhg7diw++OADjBw50uJrSpKElJQUPPLIIwgPDzdtj4uLw1//+ld8/fXXePfdd3Ho0CEMHz7crOJ0u7lz56Kqqsp0Ky0tbfFzICIiIsclfNbe7SPiJUlqdpS8pfjbt3t5eaGwsBCXL1/GV199hZSUFISFhWHYsGGNXu/VV1/Fd999h/3795ttnzx5sun/w8PDERUVhdDQUOzcuRMTJ0602DZPT094eno22XYiIiJqX4QlUv7+/nB1dW1UfSorK2tUdTLSaDQW493c3ODn52fa5uLigl69egEABgwYgKKiIqSlpTVKpH73u99hx44d+Oc//9nigltBQUEIDQ3F6dOnrX2LRERE1M4J69rz8PBAZGQkcnJyzLbn5ORgyJAhFp8THR3dKD47OxtRUVFwd3dvcl+SJJl1yUmShFdffRXbt2/H119/Da1W22J7KyoqUFpaiqCgoBZjiYiIyDkI7dpLSUnBs88+i6ioKERHR2Pt2rUoKSlBYmIigIYxR+fPn8fGjRsBNMzQS09PR0pKCmbMmAGdTof169djy5YtptdMS0tDVFQUevbsidraWmRlZWHjxo1mMwNfeeUVbN68GV988QW8vLxMVS4fHx906NABly9fRmpqKuLj4xEUFISzZ89i3rx58Pf3x4QJE9rwEyIiIiIlE5pITZ48GRUVFVi0aBH0ej3Cw8ORlZWF0NBQAIBerzdbU0qr1SIrKwvJyclYtWoVgoODsXLlSsTHx5tirly5gpkzZ+Knn35Chw4d0KdPH2zatMlszJMxqbq9q2/Dhg1ISEiAq6srjh8/jo0bN6KyshJBQUH4zW9+g61bt1q1XDzZV129hIPFF1F26ToCvNR4SNsZri68KDQREbU9oetItXe2rENB1tl1Qo+FX56Cvuq6aVuQjxoLHu+H0eHsdiUiojvnEOtIEdlq1wk9Xt501CyJAgBD1XW8vOkodp3QC2oZERE5KyZS5BDq6iUs/PIULJVPjdsWfnkKdfUssBIRUdthIkUO4WDxxUaVqFtJAPRV13Gw+GLbNYqIiJweEylyCGWXmk6i5MQRERG1BiZS5BACvNStGkdERNQamEiRQ3hI2xlBPmo0tciBCg2z9x7Sdm4igoiIqPUxkSKH4OqiwoLH+wFAo2TKeH/B4/24nhQREbUpJlLkMEaHB2H1MxHQ+Jh332l81Fj9TATXkSIiojYndGVzIluNDg/CyH4armxORESKwESKHI6riwrRPf1EN4OIiIhde0RERERyMZEiIiIikomJFBEREZFMTKSIiIiIZGIiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBMTKSIiIiKZmEgRERERycREioiIiEgmJlJEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyCU+kMjIyoNVqoVarERkZiX379jUbn5eXh8jISKjVaoSFhWHNmjVmj2/fvh1RUVG4++670alTJwwYMACffPKJzfuVJAmpqakIDg5Ghw4dMGzYMJw8efLO3zARERG1G0ITqa1btyIpKQnz589HQUEBYmJiEBcXh5KSEovxxcXFGDNmDGJiYlBQUIB58+bhtddew7Zt20wxnTt3xvz586HT6fDdd9/h+eefx/PPP4/du3fbtN9ly5bhvffeQ3p6Og4dOgSNRoORI0fi0qVL9vtAiIiIyKGoJEmSRO180KBBiIiIwOrVq03b+vbti/HjxyMtLa1R/OzZs7Fjxw4UFRWZtiUmJuLYsWPQ6XRN7iciIgJjx47F4sWLrdqvJEkIDg5GUlISZs+eDQCoqalBYGAgli5dipdeesnifmpqalBTU2O6X11djZCQEFRVVcHb29vKT4WIiIhEqq6uho+Pj1Xnb2EVqdraWhw5cgSxsbFm22NjY5Gfn2/xOTqdrlH8qFGjcPjwYdy4caNRvCRJ+Oqrr/D999/j0UcftXq/xcXFMBgMZjGenp4YOnRok20DgLS0NPj4+JhuISEhzXwCRERE5OiEJVLl5eWoq6tDYGCg2fbAwEAYDAaLzzEYDBbjb968ifLyctO2qqoq3HXXXfDw8MDYsWPxwQcfYOTIkVbv1/hfW9oGAHPnzkVVVZXpVlpa2txHQERERA7OTXQDVCqV2X1Jkhptayn+9u1eXl4oLCzE5cuX8dVXXyElJQVhYWEYNmyYTfu1tW2enp7w9PRs8nEiIiJqX4QlUv7+/nB1dW1U4SkrK2tUCTLSaDQW493c3ODn52fa5uLigl69egEABgwYgKKiIqSlpWHYsGFW7Vej0QBoqEwFBQVZ1TYiIiJyPsK69jw8PBAZGYmcnByz7Tk5ORgyZIjF50RHRzeKz87ORlRUFNzd3ZvclyRJpkHg1uxXq9VCo9GYxdTW1iIvL6/JthEREZHzEdq1l5KSgmeffRZRUVGIjo7G2rVrUVJSgsTERAANY47Onz+PjRs3AmiYoZeeno6UlBTMmDEDOp0O69evx5YtW0yvmZaWhqioKPTs2RO1tbXIysrCxo0bzWbotbRflUqFpKQkLFmyBPfccw/uueceLFmyBB07dsTUqVPb8BMiIiIiJROaSE2ePBkVFRVYtGgR9Ho9wsPDkZWVhdDQUACAXq83W9tJq9UiKysLycnJWLVqFYKDg7Fy5UrEx8ebYq5cuYKZM2fip59+QocOHdCnTx9s2rQJkydPtnq/APDGG2/g2rVrmDlzJn755RcMGjQI2dnZ8PLyaoNPhoiIiByB0HWk2jtb1qEgIiIiZXCIdaSIiIiIHB0TKSIiIiKZmEgRERERycREioiIiEgmJlJEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFMTKSIiIiIZGIiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBMTKSIiIiKZmEgRERERycREioiIiEgmJlJEREREMjGRIiIiIpLJTXQDiIjINnX1Eg4WX0TZpesI8FLjIW1nuLqoRDeLyCkxkSIiciC7Tuix8MtT0FddN20L8lFjweP9MDo8SGDLiJwTu/aIiBzErhN6vLzpqFkSBQCGqut4edNR7DqhF9QyIufFRIqIyAHU1UtY+OUpSBYeM25b+OUp1NVbiiAie2EiRUTkAA4WX2xUibqVBEBfdR0Hiy+2XaOISHwilZGRAa1WC7VajcjISOzbt6/Z+Ly8PERGRkKtViMsLAxr1qwxe3zdunWIiYmBr68vfH19MWLECBw8eNAspkePHlCpVI1ur7zyiikmISGh0eODBw9uvTdORGSDsktNJ1Fy4oiodQhNpLZu3YqkpCTMnz8fBQUFiImJQVxcHEpKSizGFxcXY8yYMYiJiUFBQQHmzZuH1157Ddu2bTPF5ObmYsqUKdi7dy90Oh26d++O2NhYnD9/3hRz6NAh6PV60y0nJwcA8OSTT5rtb/To0WZxWVlZdvgUiIhaFuClbtU4ImodKkmShHWoDxo0CBEREVi9erVpW9++fTF+/HikpaU1ip89ezZ27NiBoqIi07bExEQcO3YMOp3O4j7q6urg6+uL9PR0PPfccxZjkpKS8I9//AOnT5+GStUwhTghIQGVlZX4/PPPrX4/NTU1qKmpMd2vrq5GSEgIqqqq4O3tbfXrEBHdrq5ewiNLv4ah6rrFcVIqABofNfbPHs6lEIjuUHV1NXx8fKw6fwurSNXW1uLIkSOIjY012x4bG4v8/HyLz9HpdI3iR40ahcOHD+PGjRsWn3P16lXcuHEDnTt3brIdmzZtwvTp001JlFFubi4CAgLQu3dvzJgxA2VlZc2+p7S0NPj4+JhuISEhzcYTEVnL1UWFBY/3A9CQNN3KeH/B4/2YRBG1MWGJVHl5Oerq6hAYGGi2PTAwEAaDweJzDAaDxfibN2+ivLzc4nPmzJmDrl27YsSIERYf//zzz1FZWYmEhASz7XFxcfjrX/+Kr7/+Gu+++y4OHTqE4cOHm1Wcbjd37lxUVVWZbqWlpU3GEhHZanR4EFY/EwGNj3n3ncZHjdXPRHAdKSIBhC/IeXsVSJKkRttaire0HQCWLVuGLVu2IDc3F2q15XED69evR1xcHIKDg822T5482fT/4eHhiIqKQmhoKHbu3ImJEydafC1PT094eno22XYiojs1OjwII/tpuLI5kUIIS6T8/f3h6uraqPpUVlbWqOpkpNFoLMa7ubnBz8/PbPvy5cuxZMkS7NmzB/fff7/F1zt37hz27NmD7du3t9jeoKAghIaG4vTp0y3GEhHZk6uLCtE9/VoOJCK7E9a15+HhgcjISNOMOaOcnBwMGTLE4nOio6MbxWdnZyMqKgru7u6mbe+88w4WL16MXbt2ISoqqsk2bNiwAQEBARg7dmyL7a2oqEBpaSmCglg6JyIiogZClz9ISUnBRx99hMzMTBQVFSE5ORklJSVITEwE0DDm6NaZdomJiTh37hxSUlJQVFSEzMxMrF+/HrNmzTLFLFu2DG+++SYyMzPRo0cPGAwGGAwGXL582Wzf9fX12LBhA6ZNmwY3N/PC3OXLlzFr1izodDqcPXsWubm5ePzxx+Hv748JEybY8RMhIiIiRyJ0jNTkyZNRUVGBRYsWQa/XIzw8HFlZWQgNDQUA6PV6szWltFotsrKykJycjFWrViE4OBgrV65EfHy8KSYjIwO1tbWYNGmS2b4WLFiA1NRU0/09e/agpKQE06dPb9QuV1dXHD9+HBs3bkRlZSWCgoLwm9/8Blu3boWXl1crfwpERETkqISuI9Xe2bIOBRERESmDQ6wjRUREROTomEgRERERycREioiIiEgmJlJEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFMTKSIiIiIZGIiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBMTKSIiIiKZmEgRERERycREioiIiEgmJlJEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCbhiVRGRga0Wi3UajUiIyOxb9++ZuPz8vIQGRkJtVqNsLAwrFmzxuzxdevWISYmBr6+vvD19cWIESNw8OBBs5jU1FSoVCqzm0ajMYuRJAmpqakIDg5Ghw4dMGzYMJw8ebJ13jQRERG1C0ITqa1btyIpKQnz589HQUEBYmJiEBcXh5KSEovxxcXFGDNmDGJiYlBQUIB58+bhtddew7Zt20wxubm5mDJlCvbu3QudTofu3bsjNjYW58+fN3ut++67D3q93nQ7fvy42ePLli3De++9h/T0dBw6dAgajQYjR47EpUuXWv+DICIiIoekkiRJErXzQYMGISIiAqtXrzZt69u3L8aPH4+0tLRG8bNnz8aOHTtQVFRk2paYmIhjx45Bp9NZ3EddXR18fX2Rnp6O5557DkBDRerzzz9HYWGhxedIkoTg4GAkJSVh9uzZAICamhoEBgZi6dKleOmll6x6f9XV1fDx8UFVVRW8vb2teg4RERGJZcv5W1hFqra2FkeOHEFsbKzZ9tjYWOTn51t8jk6naxQ/atQoHD58GDdu3LD4nKtXr+LGjRvo3Lmz2fbTp08jODgYWq0WTz/9NP7973+bHisuLobBYDDbl6enJ4YOHdpk24CGZKu6utrsRkRERO2XsESqvLwcdXV1CAwMNNseGBgIg8Fg8TkGg8Fi/M2bN1FeXm7xOXPmzEHXrl0xYsQI07ZBgwZh48aN2L17N9atWweDwYAhQ4agoqLCtB/ja1vbNgBIS0uDj4+P6RYSEtJkLBERETk+4YPNVSqV2X1Jkhptayne0nagYZzTli1bsH37dqjVatP2uLg4xMfHo3///hgxYgR27twJAPj444/vqG1z585FVVWV6VZaWtpkLBERETk+N1E79vf3h6ura6MKT1lZWaNKkJFGo7EY7+bmBj8/P7Pty5cvx5IlS7Bnzx7cf//9zbalU6dO6N+/P06fPm3aD9BQmQoKCrKqbUBD95+np2ez+yIiIqL2Q1hFysPDA5GRkcjJyTHbnpOTgyFDhlh8TnR0dKP47OxsREVFwd3d3bTtnXfeweLFi7Fr1y5ERUW12JaamhoUFRWZkiatVguNRmO2r9raWuTl5TXZNiIiInI+Qrv2UlJS8NFHHyEzMxNFRUVITk5GSUkJEhMTATR0lRln2gENM/TOnTuHlJQUFBUVITMzE+vXr8esWbNMMcuWLcObb76JzMxM9OjRAwaDAQaDAZcvXzbFzJo1C3l5eSguLsa3336LSZMmobq6GtOmTQPQ0KWXlJSEJUuW4LPPPsOJEyeQkJCAjh07YurUqW306RAREZHSCevaA4DJkyejoqICixYtgl6vR3h4OLKyshAaGgoA0Ov1ZmtKabVaZGVlITk5GatWrUJwcDBWrlyJ+Ph4U0xGRgZqa2sxadIks30tWLAAqampAICffvoJU6ZMQXl5Obp06YLBgwfjwIEDpv0CwBtvvIFr165h5syZ+OWXXzBo0CBkZ2fDy8vLjp8IERERORKh60i1d1xHioiIyPE4xDpSRERERI6OiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFMTKSIiIiIZGIiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBMTKSIiIiKZmEgRERERySQrkSotLcVPP/1kun/w4EEkJSVh7dq1rdYwIiIiIqWTlUhNnToVe/fuBQAYDAaMHDkSBw8exLx587Bo0aJWbSARKU9dvQTdmQp8UXgeujMVqKuXRDeJiEgINzlPOnHiBB566CEAwN/+9jeEh4fjm2++QXZ2NhITE/GHP/yhVRtJRMqx64QeC788BX3VddO2IB81FjzeD6PDgwS2jIio7cmqSN24cQOenp4AgD179uCJJ54AAPTp0wd6vb71WkdEirLrhB4vbzpqlkQBgKHqOl7edBS7TvD7T0TORVYidd9992HNmjXYt28fcnJyMHr0aADAhQsX4Ofn16oNJCJlqKuXsPDLU7DUiWfctvDLU+zmIyKnIiuRWrp0KT788EMMGzYMU6ZMwQMPPAAA2LFjh6nLj4jal4PFFxtVom4lAdBXXcfB4ott1ygiIsFkjZEaNmwYysvLUV1dDV9fX9P23/72t+jYsWOrNY6IlKPsUtNJlJw4IqL2QFZF6tq1a6ipqTElUefOncOKFSvw/fffIyAgoFUbSETKEOClbtU4IqL2QFYiNW7cOGzcuBEAUFlZiUGDBuHdd9/F+PHjsXr16lZtIBEpw0PazgjyUUPVxOMqNMzee0jbuS2bRUQklKxE6ujRo4iJiQEA/P3vf0dgYCDOnTuHjRs3YuXKla3aQCJSBlcXFRY83g8AGiVTxvsLHu8HV5emUi0iovZHViJ19epVeHl5AQCys7MxceJEuLi4YPDgwTh37lyrNpCIlGN0eBBWPxMBjY95953GR43Vz0RwHSkicjqyBpv36tULn3/+OSZMmIDdu3cjOTkZAFBWVgZvb+9WbSARKcvo8CCM7KfBweKLKLt0HQFeDd15rEQRkTOSlUj94Q9/wNSpU5GcnIzhw4cjOjoaQEN1auDAga3aQCJSHlcXFaJ7cs04IiJZXXuTJk1CSUkJDh8+jN27d5u2P/bYY3j//fdteq2MjAxotVqo1WpERkZi3759zcbn5eUhMjISarUaYWFhWLNmjdnj69atQ0xMDHx9feHr64sRI0bg4MGDZjFpaWl48MEH4eXlhYCAAIwfPx7ff/+9WUxCQgJUKpXZbfDgwTa9NyIiImrfZCVSAKDRaDBw4EBcuHAB58+fBwA89NBD6NOnj9WvsXXrViQlJWH+/PkoKChATEwM4uLiUFJSYjG+uLgYY8aMQUxMDAoKCjBv3jy89tpr2LZtmykmNzcXU6ZMwd69e6HT6dC9e3fExsaa2gg0JGOvvPIKDhw4gJycHNy8eROxsbG4cuWK2f5Gjx4NvV5vumVlZdnyEREREVE7p5IkyebrOdTX1+Ptt9/Gu+++i8uXLwMAvLy88Pvf/x7z58+Hi4t1+dmgQYMQERFhtmRC3759MX78eKSlpTWKnz17Nnbs2IGioiLTtsTERBw7dgw6nc7iPurq6uDr64v09HQ899xzFmN+/vlnBAQEIC8vD48++iiAhopUZWUlPv/8c6veiyXV1dXw8fFBVVUVx44RERE5CFvO37IqUvPnz0d6ejr+9Kc/oaCgAEePHsWSJUvwwQcf4K233rLqNWpra3HkyBHExsaabY+NjUV+fr7F5+h0ukbxo0aNwuHDh3Hjxg2Lz7l69Spu3LiBzp2bXtumqqoKABrF5ObmIiAgAL1798aMGTNQVlbW7HuqqalBdXW12Y2IiIjaL1mDzT/++GN89NFHeOKJJ0zbHnjgAXTt2hUzZ87EH//4xxZfo7y8HHV1dQgMDDTbHhgYCIPBYPE5BoPBYvzNmzdRXl6OoKDGU6/nzJmDrl27YsSIERZfU5IkpKSk4JFHHkF4eLhpe1xcHJ588kmEhoaiuLgYb731FoYPH44jR47A09PT4mulpaVh4cKFzb5vIiIiaj9kJVIXL160OBaqT58+uHjRtguWqlTmU6YlSWq0raV4S9sBYNmyZdiyZQtyc3OhVlu+bMWrr76K7777Dvv37zfbPnnyZNP/h4eHIyoqCqGhodi5cycmTpxo8bXmzp2LlJQU0/3q6mqEhIQ0+V6IiIjIscnq2nvggQeQnp7eaHt6ejruv/9+q17D398frq6ujapPZWVljapORhqNxmK8m5sb/PzMp2IvX74cS5YsQXZ2dpNt+t3vfocdO3Zg79696NatW7PtDQoKQmhoKE6fPt1kjKenJ7y9vc1uRERE1H7JqkgtW7YMY8eOxZ49exAdHQ2VSoX8/HyUlpZaPbPNw8MDkZGRyMnJwYQJE0zbc3JyMG7cOIvPiY6Oxpdffmm2LTs7G1FRUXB3dzdte+edd/D2229j9+7diIqKavQ6kiThd7/7HT777DPk5uZCq9W22N6KigqUlpZa7D4kIiIi5ySrIjV06FD88MMPmDBhAiorK3Hx4kVMnDgRJ0+exIYNG6x+nZSUFHz00UfIzMxEUVERkpOTUVJSgsTERAANXWW3zrRLTEzEuXPnkJKSgqKiImRmZmL9+vWYNWuWKWbZsmV48803kZmZiR49esBgMMBgMJhmFwLAK6+8gk2bNmHz5s3w8vIyxVy7dg0AcPnyZcyaNQs6nQ5nz55Fbm4uHn/8cfj7+5slfUREROTkpFZUWFgoubi42PScVatWSaGhoZKHh4cUEREh5eXlmR6bNm2aNHToULP43NxcaeDAgZKHh4fUo0cPafXq1WaPh4aGSgAa3RYsWGCKsfQ4AGnDhg2SJEnS1atXpdjYWKlLly6Su7u71L17d2natGlSSUmJTe+tqqpKAiBVVVXZ9DwiIiISx5bzt6x1pJpy7NgxREREoK6urrVe0qFxHSkiIiLHY/d1pIiIiIiIiRQRERGRbDbN2mtq/SSjysrKO2kLERERkUOxKZHy8fFp8fGmrmdHRERE1N7YlEjZsrQBERERUXvHMVJEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFMTKSIiIiIZGIiRURERCQTEykiIiIimZhIEREREcnERIqIiIhIJjfRDSAiInI0dfUSDhZfRNml6wjwUuMhbWe4uqhEN4sEYCJFRERkg10n9Fj45Snoq66btgX5qLHg8X4YHR4ksGUkArv2iIiIrLTrhB4vbzpqlkQBgKHqOl7edBS7TugFtYxEYSJFRERkhbp6CQu/PAXJwmPGbQu/PIW6eksR1F4xkSIiIrLCweKLjSpRt5IA6Kuu42DxxbZrFAnHMVJERA6GA53FKLvUdBIlJ47aByZSREQOhAOdxQnwUrdqHLUP7NojInIQHOgs1kPazgjyUaOp2p8KDUntQ9rObdksEoyJFBGRA+BAZ/FcXVRY8Hg/AGiUTBnvL3i8H7tZnQwTKSIiB8CBzsowOjwIq5+JgMbHvPtO46PG6mci2L3qhDhGiojIAXCgs3KMDg/CyH4aDvgnAEykiIgcAgc6K4uriwrRPf1EN4MUgF17REQOgAOdiZSJiRQROZy6egm6MxX4ovA8dGcqnGKANQc6EymT8EQqIyMDWq0WarUakZGR2LdvX7PxeXl5iIyMhFqtRlhYGNasWWP2+Lp16xATEwNfX1/4+vpixIgROHjwoM37lSQJqampCA4ORocOHTBs2DCcPHnyzt8wEd2RXSf0eGTp15iy7gBe/7QQU9YdwCNLv3aKqf8c6EykPEITqa1btyIpKQnz589HQUEBYmJiEBcXh5KSEovxxcXFGDNmDGJiYlBQUIB58+bhtddew7Zt20wxubm5mDJlCvbu3QudTofu3bsjNjYW58+ft2m/y5Ytw3vvvYf09HQcOnQIGo0GI0eOxKVLl+z3gRBRs7iOUkMytX/2cGyZMRh/fnoAtswYjP2zhzOJIhJEJUmSsJr4oEGDEBERgdWrV5u29e3bF+PHj0daWlqj+NmzZ2PHjh0oKioybUtMTMSxY8eg0+ks7qOurg6+vr5IT0/Hc889Z9V+JUlCcHAwkpKSMHv2bABATU0NAgMDsXTpUrz00ktWvb/q6mr4+PigqqoK3t7eVj2HiCyrq5fwyNKvm1wCQIWGysz+2cPZvUV2x8v0tG+2nL+FVaRqa2tx5MgRxMbGmm2PjY1Ffn6+xefodLpG8aNGjcLhw4dx48YNi8+5evUqbty4gc6dO1u93+LiYhgMBrMYT09PDB06tMm2AQ3JVnV1tdmNiFoH11EipXDm7mVqTFgiVV5ejrq6OgQGBpptDwwMhMFgsPgcg8FgMf7mzZsoLy+3+Jw5c+aga9euGDFihNX7Nf7XlrYBQFpaGnx8fEy3kJCQJmOJHJmIwd5cR4mUgN3LdDvh60ipVOalUEmSGm1rKd7SdqBhnNOWLVuQm5sLtdp8cKY1+7W1bXPnzkVKSorpfnV1NZMpandEXTSX6yiRaC1dpkeFhsv0jOynYTefExFWkfL394erq2ujCk9ZWVmjSpCRRqOxGO/m5gY/P/OF0ZYvX44lS5YgOzsb999/v0371Wg0AGBT24CG7j9vb2+zG1F7IvLXONdRItHYvUyWCEukPDw8EBkZiZycHLPtOTk5GDJkiMXnREdHN4rPzs5GVFQU3N3dTdveeecdLF68GLt27UJUVJTN+9VqtdBoNGYxtbW1yMvLa7JtRO2d6Ivmch0lEo3dy2SJ0OUPUlJS8NFHHyEzMxNFRUVITk5GSUkJEhMTATR0lRln2gENM/TOnTuHlJQUFBUVITMzE+vXr8esWbNMMcuWLcObb76JzMxM9OjRAwaDAQaDAZcvX7Z6vyqVCklJSViyZAk+++wznDhxAgkJCejYsSOmTp3aRp8OkbIo4dc411Eikdi9TJYIHSM1efJkVFRUYNGiRdDr9QgPD0dWVhZCQ0MBAHq93mxtJ61Wi6ysLCQnJ2PVqlUIDg7GypUrER8fb4rJyMhAbW0tJk2aZLavBQsWIDU11ar9AsAbb7yBa9euYebMmfjll18waNAgZGdnw8vLy46fCJFyKeXXOC8YS6IYu5cNVdctVmaNS3Cwe9m5CF1Hqr3jOlLUnnzzYzn++6NvW4z764uD8HAv/zZoEVHbM44TBGCWTBnTeFZG2weHWEeKiByMtT+5+NOM2jF2L9PthC9/QES2EbWicvmVmlaNI3JU7F6mWzGRInIgotZwAjjQluhWri4qRPf0azmQ2j127RE5CNErKnMdJzISsbI9kVKxIkXkAJSworJxHaeXNx2FCpYH2nIdp/ZPZFWUSIlYkSJyAEpYwwngQFtnJ7oqeitWxUgpWJEicgBKWcMJ4EBbZ6WEqqgRq2KkJKxIETkApQ30Ng60HTegK6J7+jGJcgJKqYoqqSpGBDCRInIIHOhNoimhKir6eo9EljCRInIAvGAviaaEqqhSqmJEt2IiReQgONCbRFJCVVQJVTGi23GwOZEDUcpAb1Grq5M4Slj+QglVMaLbMZEicjCiV1TmjCnnZayK3n78NW10/I1VMUPVdYvjpFT/1xaOFaS2pJIkiaPy7MSWq0cTOQLjjKnb/9Ew1iDYxegcRFYkjX+DgOWqGP8GqTXYcv7mGCkisgpnTJGRyOUvOFaQlIZde0RkFVtmTPFirmRPShkrSAQwkSIiK3HGFCmJ6LGCREZMpIjIKkqaMcVZg2Lx8yf6FRMpIrKKUmZMcdagWPz8icxxsDkRWUUJq6vzOmti8fMnaoyJFBFZTeSMKSXNGqyrl6A7U4EvCs9Dd6bCKWYqKunzJ1LSd5Bde0RkE1EzppQya9BZu7aU8vkrBceJiaO07yATKSKymYgZU0qYNdjUgqTGrq32vI6REj5/pVDaidyZKPE7yK49InIIomcNOnvXlujPXyk4TkwcpX4HmUgRkUMwzhpsqvNEhYaqgL1mDdrStdUeif78lUCpJ3JnodTvIBMpInIIomcNOnvXlujPXwmUeiJ3Fkr9DjKRIoejpNka1LZEzhpk1xavc6fUE7mzUOp3kIPNyaFwkCeJmjWolAVJAbEzxpz5OndKPZE7CyV9B2/FRIochhJna5AYImYNGru2Xt50FCrA7O+wLbu2lPBjwlmvc6fUE7mzUMp38Hbs2iOHwEGedCtR3buiu7Y4Y0wsjhMTT/R30BKVJEk889hJdXU1fHx8UFVVBW9vb9HNcWi6MxWYsu5Ai3FbZgx2yl/KzkQJFRkRXWt19RIeWfp1k4OdjdWQ/bOH80RuZ0r4G3R29v4O2nL+ZtceOQQO8iRAOd27Irq2uLK4cjjzODGlUFL3MhMpcggc5Ektde+q0NC9O7Kfpl2e0JT0Y4KXR1HWiZzEYiJFDoGDPMnZKzJK+THBbi0icxxsTg6BgzxJSRUZEZSwsjgHuxM1JjyRysjIgFarhVqtRmRkJPbt29dsfF5eHiIjI6FWqxEWFoY1a9aYPX7y5EnEx8ejR48eUKlUWLFiRaPXMD52++2VV14xxSQkJDR6fPDgwa3ynkkeJc7WoLajlIqMKMYfE03NDpJg3x8TnDlLZJnQrr2tW7ciKSkJGRkZePjhh/Hhhx8iLi4Op06dQvfu3RvFFxcXY8yYMZgxYwY2bdqEb775BjNnzkSXLl0QHx8PALh69SrCwsLw5JNPIjk52eJ+Dx06hLq6OtP9EydOYOTIkXjyySfN4kaPHo0NGzaY7nt4eLTG26Y7wEGezovdu2I5e9cqUVOEJlLvvfceXnjhBbz44osAgBUrVmD37t1YvXo10tLSGsWvWbMG3bt3N1WZ+vbti8OHD2P58uWmROrBBx/Egw8+CACYM2eOxf126dLF7P6f/vQn9OzZE0OHDjXb7unpCY1Gc0fvkVofB3k6J6UuxtdWjBWhpth7sL2zd60SNUVY115tbS2OHDmC2NhYs+2xsbHIz8+3+BydTtcoftSoUTh8+DBu3Lghux2bNm3C9OnToVKZ/+OTm5uLgIAA9O7dGzNmzEBZWVmzr1VTU4Pq6mqzGxG1Hmfu3hV9wVxn71olaoqwilR5eTnq6uoQGBhotj0wMBAGg8HicwwGg8X4mzdvory8HEFBtv8j+vnnn6OyshIJCQlm2+Pi4vDkk08iNDQUxcXFeOuttzB8+HAcOXIEnp6eFl8rLS0NCxcutLkNRGS90eFBGN4nEJ/ozuLcxasI7dwRz0b3gIeb8CGfdiW6IsSuVXNcAoKMhC9/cHsVSJKkRttaire03Vrr169HXFwcgoODzbZPnjzZ9P/h4eGIiopCaGgodu7ciYkTJ1p8rblz5yIlJcV0v7q6GiEhIbLaRUSWWZp+/9H+4nY//V50RcjZu1ZvxSUg6FbCfsL5+/vD1dW1UfWprKysUdXJSKPRWIx3c3ODn5/tY2bOnTuHPXv2mMZoNScoKAihoaE4ffp0kzGenp7w9vY2uxFR63Hm6fdKWP7AmbtWjZz5b5AsE1aR8vDwQGRkJHJycjBhwgTT9pycHIwbN87ic6Kjo/Hll1+abcvOzkZUVBTc3d1tbsOGDRsQEBCAsWPHthhbUVGB0tJSWd2H1LpYUndOzr6yuVIqQs48c9bZ/wbJMqFdeykpKXj22WcRFRWF6OhorF27FiUlJUhMTATQ0FV2/vx5bNy4EQCQmJiI9PR0pKSkYMaMGdDpdFi/fj22bNlies3a2lqcOnXK9P/nz59HYWEh7rrrLvTq1csUV19fjw0bNmDatGlwczP/GC5fvozU1FTEx8cjKCgIZ8+exbx58+Dv72+W9FHbY0ndeXH6/a8Vodu/A5o2/g4468xZ/g2SJUITqcmTJ6OiogKLFi2CXq9HeHg4srKyEBoaCgDQ6/UoKSkxxWu1WmRlZSE5ORmrVq1CcHAwVq5caVr6AAAuXLiAgQMHmu4vX74cy5cvx9ChQ5Gbm2vavmfPHpSUlGD69OmN2uXq6orjx49j48aNqKysRFBQEH7zm99g69at8PLyssMnQdZQygVrSQzRg62VwpkrQqLxb5AsUUnG0drU6qqrq+Hj44OqqiqOl7pDdfUSHln6dZO/Bo0zhvbPHs4TSjulO1OBKesOtBi3ZcZgVgPILvg36DxsOX+37/nC1G6IXkOHxFPCYGtqUFcvQXemAl8UnofuTIXTXBaGf4NkifDlD4iswZI6KWWwNeDcEx6UMk5RxDFQ0t8gKQcTKXIIotfQURJnPokrYbC1UhIJEZQyTlHkMVDC3yApC8dI2RHHSLUe4xipllZVbu9jpJz5JH6r2pv1QlY2byqRMP7FtecJD0oZp6iUY+DMP2icAcdIUbtjLKkDaDQ+wVlK6lwIsMGuE3oMfWcvFu8swkbdOSzeWYSh7+y1+/tvaQ0hoGENofY6XkgJ4xSVdAyMS0CMG9AV0T392vW/PdQ8JlLkMJx5VWUlnUBEaiqZ1LdBMqmEREIkJYxTdPZjQMrEMVLkUJz1grVcCLD5ZBJo+Azsuaq0EhIJkZQwTtHZjwH9Skldq0ykyKE46wVreQJpOZkE7JtMKiGREMk49b+lcYr2nPrv7MeAGihtrGj7/hlP7YozjxHiCQQwVF1r1ThbKWkNIRHrOClhnKKSjgGJocTzABMpcghKGiMk4iTGEwhw8Uptq8bZSgmJBNBwInlk6deYsu4AXv+0EFPWHcAjS79ukxOI6HGKSjkGJIaSzgO3YtceOQSljBESVVLmQoBA57s8WzVODtFrCClhHSfR1/oTfQxIHKWcB27HRIocghLGCIk+iSnpBCJioKfG27puS2vj5BKVSFj7a9xeg+2VRHQyR2Io4TxgCRMpcgiixwi1dBJToW1OYko4gYiqyj2k7Yy7O7qj8uqNJmPu7ujeJt2bxjWE2pLowfZGShnoK+IYkFiizwNN4Rgpcgiixwgpaf0akQsBKnGg563acz3CUG3dr2xr4+RQ+vFvS8564WaRRJ8HmsJEihyC6EGmSi0ptyXRAz0PFl9sthoFAL9cvdFuF2O8eLmmVeNsJfr4K4nIAf/OTPR5oClMpMhhiJwxpNSSclsSXZUTvfyBaJ07ebRqnK1EH3+lYFVOLNEzRy3hGClyKKLGCBlLys2dSNr78gOiq3LlVlZarI1zNBqfDq0aZyvRx18JlDJW0tkpYazorZhIkcMRMcjU1UWFJx4Iwof/LG4y5okHgtr1P56iq3K/XGm+W8/WOEczIOTuVo2zlejjrwRKnX7vjJQ02YBde0RWqKuXsONY8yX7Hcf07Xp8SGSoL1rKE11UDXH2oLeyy87aOEez+dtzrRpnK2NVtjmsytoWR+0DEykiK9gy9dzeRM0WOnLuF7S0q3qpIc4egu+2rsvK2jhHc7biSqvG2cpYlW0Oq7K2xVH7wK49Iiso5ZeoyDV8RH8GQ3r6Y1XuGaviqPVZW5V9Y3TfdptMKeHCzaQ8rEgRWUEJv0RFzxYS/RkM7umHuzu6Nxtzd0d3DFbIuInWNiDEui5Ta+NspaSqLOC8F24m5WEiRWQF0eNDlLCGj+gxUq4uKvxpYv9mY/40sX+7PYmJvkSO6IrkrZz5ws2kPOzac0AirnPm7ETP2lPCbCFbxkjZqw2jw4Ow5pkILPjiBP5zqda0PdDLAwvHhbfZSUzEd7Besi5JtjbOVqIrkkair3kJKG/6PYnFRMrBKOU6V85G9PgQJVQDlHCJEiOVyqXZ+/Yk6jv4bXGF1XExvbu0+v6VMD5ISes4iZ5+zx/UysGuPQfS1BgZPVfUtTvR40OUUA0QfYkS4NfvwO3J2n+q2+Y7IHKcmrV1Jnt17iphfBBXV2/AS9QoCxMpB9HcLzGg4R8QZ7nOlQiiK0JKuFin6EuUiB4nJnr/Pp7ND7S3NU4O0eODRH8PbyVqGRLRk06oMXbtOQhbKiJKWe21PRFdETJWA17edBQqmFcd2qoaIPoSJaLHiYnef+X12paDbIiTS+T4INHfQyNR3btK6tqkX7Ei5SCc/YKtoj2k7dzi1Hvfju52rQiJrgZY8xncbcfPQHQ1QvT+XawcB2Zt3J0wjg8aN6Aronv6tdlJWwmVWZEVIXZtKhMrUg7i4hXrfmVaG0etry0K+0qfLWTPVoiuRvh38mzVOFtF9/RD+t4frYprr0RXZkVXhEQn82QZK1IOovNd1v3jbG0c2eZg8UVUXm3+YriVV2+0yS9BUdUAaz6DX+z4GQivRlj7MdvpcAwOa3lBUt+O7hgc1n4TKUBsZVZ0RUj0jwmyjBUpByF6MT5np6Sp/6KI/jVsrEYkbjpq8XEJ9q1GlFs5G9HaOFu5uqgwOapbs2uZPRXVTTHVSXsSVZkV/R0wLorb3Lh2ey6KS5axIuUgRK+s7ezKrfyH0do4R+RvZbXT2jhHI7prr65ewqYDJc3GbPq2xGlm7oqozIquCIm+cDhZxkTKQRh/jatgeQ0XFXiNJ3tqqUvL1jiHJHghI+P4lKYYx6fYLZEQ3LWX/2M5rtTWNRtzpaYO+T+W26cBJLx7WXRFjCxjIuVARM/acmYqlXVnR2vjHFH5FSu7tqyMs5Xo8Smiu/a2H/2pVePIdqIXJRVdESPLhCdSGRkZ0Gq1UKvViIyMxL59+5qNz8vLQ2RkJNRqNcLCwrBmzRqzx0+ePIn4+Hj06NEDKpUKK1asaPQaqampUKlUZjeNRmMWI0kSUlNTERwcjA4dOmDYsGE4efLkHb/fOzU6PAj7Zw/HlhmD8eenB2DLjMHYP3u4UyVRIhbCs3YmVHueMSX6H3HRv8ZFd222VI2yNe5OiFqMUglE/qAVXREjy4QONt+6dSuSkpKQkZGBhx9+GB9++CHi4uJw6tQpdO/evVF8cXExxowZgxkzZmDTpk345ptvMHPmTHTp0gXx8fEAgKtXryIsLAxPPvkkkpOTm9z3fffdhz179pjuu7q6mj2+bNkyvPfee/jLX/6C3r174+2338bIkSPx/fffw8vLq5U+AXlEX+NJJFEL4RlnTDXXddfeZ0yJvtaa6EROdNfmgz18kX3qP1bF2ROv9ylusLvo5R9uxWv9/UpoReq9997DCy+8gBdffBF9+/bFihUrEBISgtWrV1uMX7NmDbp3744VK1agb9++ePHFFzF9+nQsX77cFPPggw/inXfewdNPPw1Pz6Z/Gbq5uUGj0ZhuXbr8epFPSZKwYsUKzJ8/HxMnTkR4eDg+/vhjXL16FZs3b269D4BsInIhPFcXFf40sX+zMWkT+7frf0hEd2sYZyw1x54zlkR3bU4bom1x+JXq/+LsRUmXJxFdFRO1DIkShnjwWn/mhCVStbW1OHLkCGJjY822x8bGIj8/3+JzdDpdo/hRo0bh8OHDuHHDtkG+p0+fRnBwMLRaLZ5++mn8+9//Nj1WXFwMg8Fgti9PT08MHTq0ybYBQE1NDaqrq81u1DpEX+cMaPgHbM0zEY2WmAjyUWONk4xRE/mPuOgZS6IrYh5uLvjto80nSb99VAsPN/v8s66E76CRs5/IRQ7xUFIyrRTCuvbKy8tRV1eHwMBAs+2BgYEwGAwWn2MwGCzG37x5E+Xl5QgKsu6PaNCgQdi4cSN69+6N//znP3j77bcxZMgQnDx5En5+fqb9W9rXuXPnmnzdtLQ0LFy40Ko2kG1EX+fMSOkri7cFZ13DR3TXJgAM7O4LoOl1pBoetw+lfAeNJ/Lbj4HxRO4sE29EDPEQvbK7UgkfbH77LCdJkpqd+WQp3tL25sTFxSE+Ph79+/fHiBEjsHPnTgDAxx9/fEdtmzt3Lqqqqky30tJSq9tEzRN9Er2VqJK+sxNdEbq1a7MpbXF5kqbYe/kHJXwHlVQVc0aiZ84qlbCKlL+/P1xdXRtVn8rKyhpVgow0Go3FeDc3N/j5yc/MO3XqhP79++P06dOm/QANFbBbq1zNtQ1o6P5rblwWySf6JEq/2nVCj9QdJ2Go/nUskMbbE6lP3GfXSoASVnUeHR6E3z6qxbp9xWbtcFEBM2K0irk8iT0qFUr4Dor+DJydEpJpJRJWkfLw8EBkZCRycnLMtufk5GDIkCEWnxMdHd0oPjs7G1FRUXB3b/4aVM2pqalBUVGRKWnSarXQaDRm+6qtrUVeXl6TbSP74rRfZdh1Qo/ETUfNkigAMFTXINHO4yNEj5ECGt7/2n8WN2qHJAFr/1ls1/cv+iSmhO+g6M/A2SkhmVYioV17KSkp+Oijj5CZmYmioiIkJyejpKQEiYmJABq6yp577jlTfGJiIs6dO4eUlBQUFRUhMzMT69evx6xZs0wxtbW1KCwsRGFhIWpra3H+/HkUFhbixx9/vWr6rFmzkJeXh+LiYnz77beYNGkSqqurMW3aNAANXXpJSUlYsmQJPvvsM5w4cQIJCQno2LEjpk6d2kafDt1K9IwxauhWmbP9eLMxc7Yfb7ddS6K7lUSvY6WE7yBP5GIpIZlWIqGJ1OTJk7FixQosWrQIAwYMwD//+U9kZWUhNDQUAKDX61FS8uu1pbRaLbKyspCbm4sBAwZg8eLFWLlypWkNKQC4cOECBg4ciIEDB0Kv12P58uUYOHAgXnzxRVPMTz/9hClTpuDee+/FxIkT4eHhgQMHDpj2CwBvvPEGkpKSMHPmTERFReH8+fPIzs4WvoaUM1PCtF8lEDXt+8CZihYvgVN59QYOnKmwy/5Fn0SFjw8RvI4VIP47KHoJDGenhGRaiVSScbQ2tbrq6mr4+PigqqoK3t7eopvTbjjzQnAiF0NcvvtfSN97psW4V3/TE7NG9Wn1/dfVS3hk6dctzprbP3u4Xf4evig8j9c/LWwx7s9PD8C4AV1bff+fFZxH8taW9//+5AGYMLD1938rUd9B3ZkKTFl3oMW4LTMGc4yUHTnDoqy2nL+FrmxOJIezruwuftq32Kv2il7VWXRF7KKV1/CzNu5OiPoOiu7epQajw4MwvE8gPtGdxbmLVxHauSOeje5htzXMLFHSD2omUkQOQAnrt0T39EP63h+tirMXY9fS7b+GNW3wa1j0OlJ3d7BuQo21cY5IdDJLDSxVpD7aX9xmFSmlVcSYSBE5ACVM+1bK9Qad9Tpnldesu3qDtXGOSHQyS+Ir46L3b4nwBTmJqGVK6NJQ0vUGRV/nLNC77Qdbd7ZyNp61cY5ISYOdRV/rTwTRM1dF778pTKSIHIBSujR4vUEj83+o22LOToCXdQmStXGOSvTMQcB5r/Uneuaq6P03hV17RA5ASV0azny9waa6Ff5TXWP/bgUFLH9gJHqgr8i/QSV2LbUV0ZVx0ftvChMpIgcgenyOpfY428xJ0QP+y69YNxvP2ji5lDLQlxftbXuiK+Oi998Udu0ROQgldGk4M9HdCko4iRirMbd/DsZqDLu22vdFe0WvbC56/01hRYrIgThzt5poorsVRHfvOns1BhD/NyCa6Mq46P03hRUpIgcjasaasxNdERI9Y83ZqzGA+L8BJRBdGRe9f0tYkSIisoLoihAgdkFSZ6/GAMr4G1AC0ZVx0fu/HRMpIiIrKKVbQdRJhNUY5fwNKIHoCSei938rdu2Rw3HGhfBIGZTSrSCie1epA33bmlL+Bkg5VFJbrCTnpGy5ejRZRylTr8m5iV5HSRTjrD3AcjXGmRIJZ/0bcBa2nL+ZSNkRE6nW1dRCeM74jziRKPwxQ87AlvM3x0iRQ+DUayJlUNpAXyLRmEiRQ7Bl6rVSBiAStVdKGuhLJBoHm5ND4NRrIiJSIiZS5BA49ZqIiJSIiRQ5BE69JiIiJWIiRQ5B9OUxiIiILGEiRQ6DC+EREZHScNYeORROvSYiIiVhIkUOh1OviYhIKdi1R0RERCQTEykiIiIimZhIEREREcnERIqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBNXNrcjSZIAANXV1YJbQkRERNYynreN5/HmMJGyo0uXLgEAQkJCBLeEiIiIbHXp0iX4+Pg0G6OSrEm3SJb6+npcuHABXl5euHTpEkJCQlBaWgpvb2/RTaNbVFdX89goFI+NcvHYKBuPz52RJAmXLl1CcHAwXFyaHwXFipQdubi4oFu3bgAAlUoFAPD29uYftULx2CgXj41y8dgoG4+PfC1Voow42JyIiIhIJiZSRERERDIxkWojnp6eWLBgATw9PUU3hW7DY6NcPDbKxWOjbDw+bYeDzYmIiIhkYkWKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFMTKRaUUZGBrRaLdRqNSIjI7Fv374mY7dv346RI0eiS5cu8Pb2RnR0NHbv3t2GrXUuthyb/fv34+GHH4afnx86dOiAPn364P3332/D1joXW47Nrb755hu4ublhwIAB9m2gE7Pl2OTm5kKlUjW6/etf/2rDFjsPW783NTU1mD9/PkJDQ+Hp6YmePXsiMzOzjVrbzknUKj799FPJ3d1dWrdunXTq1Cnp9ddflzp16iSdO3fOYvzrr78uLV26VDp48KD0ww8/SHPnzpXc3d2lo0ePtnHL2z9bj83Ro0elzZs3SydOnJCKi4ulTz75ROrYsaP04YcftnHL2z9bj41RZWWlFBYWJsXGxkoPPPBA2zTWydh6bPbu3SsBkL7//ntJr9ebbjdv3mzjlrd/cr43TzzxhDRo0CApJydHKi4ulr799lvpm2++acNWt19MpFrJQw89JCUmJppt69OnjzRnzhyrX6Nfv37SwoULW7tpTq81js2ECROkZ555prWb5vTkHpvJkydLb775prRgwQImUnZi67ExJlK//PJLG7TOudl6bP73f/9X8vHxkSoqKtqieU6HXXutoLa2FkeOHEFsbKzZ9tjYWOTn51v1GvX19bh06RI6d+5sjyY6rdY4NgUFBcjPz8fQoUPt0USnJffYbNiwAWfOnMGCBQvs3USndSffm4EDByIoKAiPPfYY9u7da89mOiU5x2bHjh2IiorCsmXL0LVrV/Tu3RuzZs3CtWvX2qLJ7R4vWtwKysvLUVdXh8DAQLPtgYGBMBgMVr3Gu+++iytXruCpp56yRxOd1p0cm27duuHnn3/GzZs3kZqaihdffNGeTXU6co7N6dOnMWfOHOzbtw9ubvzny17kHJugoCCsXbsWkZGRqKmpwSeffILHHnsMubm5ePTRR9ui2U5BzrH597//jf3790OtVuOzzz5DeXk5Zs6ciYsXL3KcVCvgv0StSKVSmd2XJKnRNku2bNmC1NRUfPHFFwgICLBX85yanGOzb98+XL58GQcOHMCcOXPQq1cvTJkyxZ7NdErWHpu6ujpMnToVCxcuRO/evduqeU7Nlu/Nvffei3vvvdd0Pzo6GqWlpVi+fDkTKTuw5djU19dDpVLhr3/9K3x8fAAA7733HiZNmoRVq1ahQ4cOdm9ve8ZEqhX4+/vD1dW10a+BsrKyRr8abrd161a88MIL+J//+R+MGDHCns10SndybLRaLQCgf//++M9//oPU1FQmUq3I1mNz6dIlHD58GAUFBXj11VcBNJwgJEmCm5sbsrOzMXz48DZpe3t3J9+bWw0ePBibNm1q7eY5NTnHJigoCF27djUlUQDQt29fSJKEn376Cffcc49d29zecYxUK/Dw8EBkZCRycnLMtufk5GDIkCFNPm/Lli1ISEjA5s2bMXbsWHs30ynJPTa3kyQJNTU1rd08p2brsfH29sbx48dRWFhouiUmJuLee+9FYWEhBg0a1FZNb/da63tTUFCAoKCg1m6eU5NzbB5++GFcuHABly9fNm374Ycf4OLigm7dutm1vU5B3Dj39sU4HXX9+vXSqVOnpKSkJKlTp07S2bNnJUmSpDlz5kjPPvusKX7z5s2Sm5ubtGrVKrOpwpWVlaLeQrtl67FJT0+XduzYIf3www/SDz/8IGVmZkre3t7S/PnzRb2FdsvWY3M7ztqzH1uPzfvvvy999tln0g8//CCdOHFCmjNnjgRA2rZtm6i30G7ZemwuXbokdevWTZo0aZJ08uRJKS8vT7rnnnukF198UdRbaFeYSLWiVatWSaGhoZKHh4cUEREh5eXlmR6bNm2aNHToUNP9oUOHSgAa3aZNm9b2DXcCthyblStXSvfdd5/UsWNHydvbWxo4cKCUkZEh1dXVCWh5+2fLsbkdEyn7suXYLF26VOrZs6ekVqslX19f6ZFHHpF27twpoNXOwdbvTVFRkTRixAipQ4cOUrdu3aSUlBTp6tWrbdzq9kklSZIksiJGRERE5Kg4RoqIiIhIJiZSRERERDIxkSIiIiKSiYkUERERkUxMpIiIiIhkYiJFREREJBMTKSIiIiKZmEgRERERycREiogczrBhw5CUlGR1/NmzZ6FSqVBYWGi3NrXlfpqSm5sLlUqFyspKIfsnckZMpIhIuISEBKhUKiQmJjZ6bObMmVCpVEhISDBt2759OxYvXmz164eEhECv1yM8PLw1mntHhg0bBpVKBZVKBQ8PD/Ts2RNz5861+aLYlpLJIUOGQK/Xw8fHpxVbTETNYSJFRIoQEhKCTz/9FNeuXTNtu379OrZs2YLu3bubxXbu3BleXl5Wv7arqys0Gg3c3Nxarb13YsaMGdDr9fjxxx+xbNkyrFq1CqmpqXf8uh4eHtBoNFCpVHfeSCKyChMpIlKEiIgIdO/eHdu3bzdt2759O0JCQjBw4ECz2NurMT169MCSJUswffp0eHl5oXv37li7dq3p8du73IxdYLt378bAgQPRoUMHDB8+HGVlZfjf//1f9O3bF97e3pgyZQquXr1qep1du3bhkUcewd133w0/Pz/813/9F86cOWPze+3YsSM0Gg26d++O+Ph4jBw5EtnZ2abHKyoqMGXKFHTr1g0dO3ZE//79sWXLFtPjCQkJyMvLw5///GdTdevs2bMWu/a2bduG++67D56enujRowfeffddm9tLRE1jIkVEivH8889jw4YNpvuZmZmYPn26Vc999913ERUVhYKCAsycORMvv/wy/vWvfzX7nNTUVKSnpyM/Px+lpaV46qmnsGLFCmzevBk7d+5ETk4OPvjgA1P8lStXkJKSgkOHDuGrr76Ci4sLJkyYgPr6enlvGMCxY8fwzTffwN3d3bTt+vXriIyMxD/+8Q+cOHECv/3tb/Hss8/i22+/BQD8+c9/RnR0tKmypdfrERIS0ui1jxw5gqeeegpPP/00jh8/jtTUVLz11lv4y1/+Iru9RHQbiYhIsGnTpknjxo2Tfv75Z8nT01MqLi6Wzp49K6nVaunnn3+Wxo0bJ02bNs0UP3ToUOn111833Q8NDZWeeeYZ0/36+nopICBAWr16tSRJklRcXCwBkAoKCiRJkqS9e/dKAKQ9e/aYnpOWliYBkM6cOWPa9tJLL0mjRo1qst1lZWUSAOn48eMW92PJ0KFDJXd3d6lTp06Sh4eHBEBycXGR/v73vzf7GY0ZM0b6/e9/3+RncOv7+uWXXyRJkqSpU6dKI0eONIv5f//v/0n9+vVrdl9EZD1WpIhIMfz9/TF27Fh8/PHH2LBhA8aOHQt/f3+rnnv//feb/l+lUkGj0aCsrMzq5wQGBqJjx44ICwsz23bra5w5cwZTp05FWFgYvL29odVqAQAlJSVWtdHov//7v1FYWAidToennnoK06dPR3x8vOnxuro6/PGPf8T9998PPz8/3HXXXcjOzrZ5P0VFRXj44YfNtj388MM4ffo06urqbHotIrJMGSMviYj+z/Tp0/Hqq68CAFatWmX1827tGgMakqmWutxufY5KpWrxNR5//HGEhIRg3bp1CA4ORn19PcLDw1FbW2t1OwHAx8cHvXr1AgBs2rQJ9913H9avX48XXngBQEM35fvvv48VK1agf//+6NSpE5KSkmzejyRJjQaeS5Jk02sQUfNYkSIiRRk9ejRqa2tRW1uLUaNGiW6OSUVFBYqKivDmm2/iscceQ9++ffHLL7/c8eu6u7tj3rx5ePPNN00D2/ft24dx48bhmWeewQMPPICwsDCcPn3a7HkeHh4tVpX69euH/fv3m23Lz89H79694erqesdtJyImUkSkMK6urigqKkJRUZGiTva+vr7w8/PD2rVr8eOPP+Lrr79GSkpKq7z21KlToVKpkJGRAQDo1asXcnJykJ+fj6KiIrz00kswGAxmz+nRowe+/fZbnD17FuXl5Rarb7///e/x1VdfYfHixfjhhx/w8ccfIz09HbNmzWqVdhMREykiUiBvb294e3uLboYZFxcXfPrppzhy5AjCw8ORnJyMd955p1Ve28PDA6+++iqWLVuGy5cv46233kJERARGjRqFYcOGQaPRYPz48WbPmTVrFlxdXdGvXz906dLF4vipiIgI/O1vf8Onn36K8PBw/OEPf8CiRYvMFjclojujkthhTkRERCQLK1JEREREMjGRIiIiIpKJiRQRERGRTEykiIiIiGRiIkVEREQkExMpIiIiIpmYSBERERHJxESKiIiISCYmUkREREQyMZEiIiIikomJFBEREZFM/x9nU0Q1/5ClYwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(minRatio, Loss)\n",
    "plt.xlabel('Minimal Ratio')\n",
    "plt.ylabel('Loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ce6e3e1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(simulatedInput)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "570c3064",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}