experiment_seer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3f1281c7",
   "metadata": {},
   "source": [
    "# Run SurvTRACE on SEER dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "afd9a31d",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''SEER data comes from https://seer.cancer.gov/data/\n",
    "'''\n",
    "from survtrace.dataset import load_data\n",
    "from survtrace.evaluate_utils import Evaluator\n",
    "from survtrace.utils import set_random_seed\n",
    "from survtrace.model import SurvTraceMulti\n",
    "from survtrace.train_utils import Trainer\n",
    "from survtrace.config import STConfig\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# define the setup parameters\n",
    "STConfig['data'] = 'seer'\n",
    "STConfig['num_hidden_layers'] = 2\n",
    "STConfig['hidden_size'] = 16\n",
    "STConfig['intermediate_size'] = 64\n",
    "STConfig['num_attention_heads'] = 2\n",
    "STConfig['initializer_range'] = .02\n",
    "STConfig['early_stop_patience'] = 5\n",
    "set_random_seed(STConfig['seed'])\n",
    "\n",
    "hparams = {\n",
    "    'batch_size': 1024,\n",
    "    'weight_decay': 0,\n",
    "    'learning_rate': 1e-4,\n",
    "    'epochs': 100,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1a1b74d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "df, df_train, df_y_train, df_test, df_y_test, df_val, df_y_val = load_data(STConfig)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a214b11",
   "metadata": {},
   "outputs": [],
   "source": [
    "# get model\n",
    "model = SurvTraceMulti(STConfig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdb39652",
   "metadata": {},
   "source": [
    "## kick off the training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0f3cf177",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use pytorch-cuda for training.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\UIUC_AI_Health\\SurvTRACE\\survtrace\\train_utils.py:208: UserWarning: This overload of add_ is deprecated:\n",
      "\tadd_(Number alpha, Tensor other)\n",
      "Consider using one of the following signatures instead:\n",
      "\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ..\\torch\\csrc\\utils\\python_arg_parser.cpp:1025.)\n",
      "  next_m.mul_(beta1).add_(1 - beta1, grad)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train-0]: 2.9641945240448933\n",
      "[Val-0]: 1.9814077615737915\n",
      "[Train-1]: 0.7204283676585372\n",
      "[Val-1]: 0.6945959329605103\n",
      "[Train-2]: 0.6825504732780716\n",
      "[Val-2]: 0.6849672794342041\n",
      "[Train-3]: 0.6769993368054734\n",
      "[Val-3]: 0.6842683553695679\n",
      "[Train-4]: 0.6750199766386122\n",
      "[Val-4]: 0.6799230575561523\n",
      "[Train-5]: 0.67327523515338\n",
      "[Val-5]: 0.6782727837562561\n",
      "[Train-6]: 0.6719713884146035\n",
      "[Val-6]: 0.6769348382949829\n",
      "[Train-7]: 0.6710905238073699\n",
      "[Val-7]: 0.6765552759170532\n",
      "[Train-8]: 0.6704727275436427\n",
      "[Val-8]: 0.6768720149993896\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-9]: 0.669449763030422\n",
      "[Val-9]: 0.6756874918937683\n",
      "[Train-10]: 0.6691256964287791\n",
      "[Val-10]: 0.6752216219902039\n",
      "[Train-11]: 0.6683859139883599\n",
      "[Val-11]: 0.6739037036895752\n",
      "[Train-12]: 0.6683868603641484\n",
      "[Val-12]: 0.6749087572097778\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-13]: 0.6675718284788585\n",
      "[Val-13]: 0.673507571220398\n",
      "[Train-14]: 0.6674971448726394\n",
      "[Val-14]: 0.673939049243927\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-15]: 0.6667733137704888\n",
      "[Val-15]: 0.6734627485275269\n",
      "[Train-16]: 0.6666211033759474\n",
      "[Val-16]: 0.6732335090637207\n",
      "[Train-17]: 0.6663265518185233\n",
      "[Val-17]: 0.672447144985199\n",
      "[Train-18]: 0.6664419212714344\n",
      "[Val-18]: 0.6743324995040894\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-19]: 0.6662146931602841\n",
      "[Val-19]: 0.6738592982292175\n",
      "EarlyStopping counter: 2 out of 5\n",
      "[Train-20]: 0.6661289859385717\n",
      "[Val-20]: 0.672224760055542\n",
      "[Train-21]: 0.6661100199027937\n",
      "[Val-21]: 0.6752310395240784\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-22]: 0.6655362311269151\n",
      "[Val-22]: 0.6727402210235596\n",
      "EarlyStopping counter: 2 out of 5\n",
      "[Train-23]: 0.6655360756277227\n",
      "[Val-23]: 0.6720191836357117\n",
      "[Train-24]: 0.6648161727554944\n",
      "[Val-24]: 0.6719292402267456\n",
      "[Train-25]: 0.665206140604149\n",
      "[Val-25]: 0.6733843088150024\n",
      "EarlyStopping counter: 1 out of 5\n",
      "[Train-26]: 0.6645853059226964\n",
      "[Val-26]: 0.6726152896881104\n",
      "EarlyStopping counter: 2 out of 5\n",
      "[Train-27]: 0.6648328999678293\n",
      "[Val-27]: 0.6720612645149231\n",
      "EarlyStopping counter: 3 out of 5\n",
      "[Train-28]: 0.6644353844276091\n",
      "[Val-28]: 0.6724880933761597\n",
      "EarlyStopping counter: 4 out of 5\n",
      "[Train-29]: 0.6641093496562672\n",
      "[Val-29]: 0.6723611950874329\n",
      "EarlyStopping counter: 5 out of 5\n",
      "early stops at epoch 30\n"
     ]
    }
   ],
   "source": [
    "# initialize a trainer & start training\n",
    "trainer = Trainer(model)\n",
    "train_loss_list, val_loss_list = trainer.fit((df_train, df_y_train), (df_val, df_y_val),\n",
    "        batch_size=hparams['batch_size'],\n",
    "        epochs=hparams['epochs'],\n",
    "        learning_rate=hparams['learning_rate'],\n",
    "        weight_decay=hparams['weight_decay'],\n",
    "        val_batch_size=10000,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "95fae954",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******************************\n",
      "start evaluation\n",
      "******************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\anaconda3\\envs\\survtrace\\lib\\site-packages\\sksurv\\metrics.py:134: RuntimeWarning: overflow encountered in long_scalars\n",
      "  concordant += n_con\n",
      "e:\\anaconda3\\envs\\survtrace\\lib\\site-packages\\sksurv\\metrics.py:134: RuntimeWarning: overflow encountered in long_scalars\n",
      "  concordant += n_con\n",
      "e:\\anaconda3\\envs\\survtrace\\lib\\site-packages\\sksurv\\metrics.py:134: RuntimeWarning: overflow encountered in long_scalars\n",
      "  concordant += n_con\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event: 0 For 0.25 quantile,\n",
      "TD Concordance Index - IPCW: 0.9054360446472517\n",
      "Brier Score: 0.03556944931818888\n",
      "Event: 0 For 0.5 quantile,\n",
      "TD Concordance Index - IPCW: 0.8831556261217498\n",
      "Brier Score: 0.06044895229135169\n",
      "Event: 0 For 0.75 quantile,\n",
      "TD Concordance Index - IPCW: 0.8652413483065683\n",
      "Brier Score: 0.08204407514522072\n",
      "Event: 1 For 0.25 quantile,\n",
      "TD Concordance Index - IPCW: 0.7997237378939374\n",
      "Brier Score: 0.007855027637038457\n",
      "Event: 1 For 0.5 quantile,\n",
      "TD Concordance Index - IPCW: 0.7895639461300388\n",
      "Brier Score: 0.015905955178805307\n",
      "Event: 1 For 0.75 quantile,\n",
      "TD Concordance Index - IPCW: 0.7722955042907839\n",
      "Brier Score: 0.027555113590624722\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "# evaluate model\n",
    "evaluator = Evaluator(df, df_train.index)\n",
    "evaluator.eval(model, (df_test, df_y_test))\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "97b7a495",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEPCAYAAABRHfM8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmYklEQVR4nO3de5gcdZ3v8fe3u2d6knQnZMjMJHILCCgRSWCziAbXLBeXy4LickAPy4L4HNh9ZA8Xz1GXfdCwsiKryy67LCJnUcALICiLcECEQ8JlFWK4iRB0o8YIBHOZXCYJmUv39/xR1T01PT0zPZ2Zqempz+t5+qlrV32reqa//atf1e9n7o6IiMhIUnEHICIijUEJQ0REaqKEISIiNVHCEBGRmihhiIhITTJxBzAe5syZ4/Pnz487DBGRhvLss89ucve2oZZPyYQxf/58Vq1aFXcYIiINxcx+O9xyXZISEZGaKGGIiEhNYk0YZtZiZivN7EUze9nMrqqyTtbM7jKzNWb2jJnNjyFUEZHEi7uE0Q0c5+4LgUXASWZ2TMU6nwC2uPvBwD8B105siCIiAjEnDA/sCCebwldl41YfAm4Lx+8Bjjczm6AQRUQkFHcJAzNLm9kLwAbgEXd/pmKVfYDfAbh7H7AN2LvKdi40s1Vmtmrjxo3jHLWISPLEnjDcveDui4B9gaPN7PA6t3Ozuy9298VtbUPeRiwiInWKPWGUuPtWYDlwUsWi14H9AMwsA8wCNo9HDL94s4uvPPwLOnf2jMfmRUQaWqwP7plZG9Dr7lvNbBpwIoMrtX8AnAf8BDgTeMzHqROP32zawQ3L13DqEfNondE8HrsQmdS6u7vp7Oykq6uLQqEQdziyB9LpNPl8ntbWVrLZ7JhsM+4nvecBt5lZmqC08113f8DM/g5Y5e4/AG4Bvmlma4BO4KPjFUwu2wTAju6+8dqFyKTV3d3NunXrmD17NvPnz6epqQndX9KY3J3e3l62b9/OunXr2H///cckacSaMNz9Z8CRVeZ/LjK+G/hvExFPriU4HTt2K2FI8nR2djJ79mzmzJkTdyiyh8yM5ubm8mfZ2dnJvHnz9ni7k6YOYzLIZYOE0aUShiRQV1cXM2fOjDsMGWMzZ86kq6trTLalhBGRVwlDEqxQKNDU1BR3GDLGmpqaxqw+SgkjolTC2NHdG3MkIvFQncXUM5afqRJGxPTmNGYqYYiIVKOEEWFm5JozqsMQEalCCaNCriWjEoaISBVKGBVy2YyewxCRCbNs2TLMjBUrVsQdyoiUMCrkWpQwRJJs7dq1mBnnn39+3KFMOkoYFVTCEJGJdPHFF7N69WqOPvrouEMZUdxNg0w6+ZYMb27bHXcYIpIQc+bMaZin61XCqKAShkhyLVu2jAMPPBCA2267DTMrv2699VZWrFiBmbFs2TJWrlzJqaeeSmtrK2bG2rVrAVi+fDkXXnghCxYsYObMmUybNo3DDz+cq666it27B/8YHaoOw8xYunQpmzZt4sILL2TevHlks1ne9a538Y1vfGO8T0VVKmFUyGWbdJeUSEItXbqUrVu3cv3117Nw4UI+/OEPl5ctWrSIrVu3AvCTn/yEa665hmOPPZYLLriATZs20dwctHB97bXX8uqrr/K+972PU089ld27d/Of//mfLFu2jBUrVvDoo4+STqdrimfr1q0sWbKE5uZmzjzzTLq7u7n77ru54IILSKVSnHfeeWN9CoalhFEh15JhR08fxaKTSumpVxGAq+5/mVfe2B53GMNa8LaZfP60d+3RNpYuXcr8+fO5/vrrWbRoEcuWLRuwvFQK+NGPfsRNN93ERRddNGgbN954IwceeOCgJ6yvvPJKrr76au655x7OPvvsmuJ58cUX+cQnPsHXvva1cpK59NJLOeKII7j22msnPGHoklSFfDaDO+zqVV8AIlLdokWLqiYLgIMOOqhqcxyXXXYZAA8//HDN+5k+fTrXXXfdgBLJggULWLJkCatXr2bHjh2jjHzPqIRRIdrEealtKZGk29Nf7lPNcHc07dy5k+uvv557772XX/7yl3R1dRHt8+3111+veT+HHHJI1RaE99tvPwC2bNlCLpcbReR7Rt+IFQY2QNgSbzAiMinNnTu36vze3l6OO+44Vq5cyeGHH87ZZ59NW1tbuRXgq666iu7u7pr3s9dee1Wdn8kE31MT3SuiEkaFUgmjSxXfIjKEoVqAve+++1i5ciXnn3/+oDuZ1q9fz1VXXTUR4Y0b1WFUyJdLGEoYIklUqi+o59f7mjVrAPjIRz4yaNnjjz++Z4FNAkoYFWZk1YmSSJLNnj0bM2PdunWjfu/8+fMBBj1T8etf/5rPfOYzYxBdvHRJqoK6aRVJtlwux3ve8x6efPJJzjnnHA499FDS6TSnn376iO897bTTOPjgg7nuuut46aWXOPLII1m3bh0PPPAAp556al1JaDJRwqigblpF5Jvf/CaXXXYZP/zhD7njjjtwd/bdd99yCWIoM2bM4LHHHuOzn/0sK1as4Mknn+Sggw7iyiuv5PLLL+euu+6amAMYJxa93WuqWLx4sa9ataqu9/YWihzytw9x+YmH8j+PP2SMIxOZvFavXs1hhx0WdxgyDmr9bM3sWXdfPNRy1WFUaEqnaGlKsVOXpEREBlDCqCKXbVIdhohIBSWMKvLqplVEZBAljCrUxLmIyGBKGFXksiphiIhUUsKoIteSUR2GiEgFJYwq8tlM2PigiIiUKGFUkVOlt4jIIEoYVZQqvafiQ40iIvVSwqgi15Kht+B09xXjDkVEZNJQwqhCTZyLiAymhFGFmjgXERks1oRhZvuZ2XIze8XMXjazS6qss9TMtpnZC+Hrc+MdV04lDBGRQeJu3rwP+JS7P2dmeeBZM3vE3V+pWO9Jd//TiQpK3bSKyHgpNZG+du3aWOOoR6wlDHdf7+7PheNdwGpgnzhjAshngw7bVcIQEek3aeowzGw+cCTwTJXF7zWzF83sITN71xDvv9DMVpnZqo0bN+5RLKUShpo4FxHpNykShpnlgO8Bl7r79orFzwEHuPtC4F+B/6i2DXe/2d0Xu/vitra2PYpH3bSKiAwWe8IwsyaCZPFtd/9+5XJ33+7uO8LxB4EmM5sznjGpm1aRZHr66acxM84444wh1znssMPIZrN0dnbS09PDDTfcwCmnnMIBBxxANpultbWVE044gYceemgCI58Ycd8lZcAtwGp3v26IdeaG62FmRxPEvHk848pmUmRSpvakRBLmmGOO4R3veAcPPvggmzcP/ppZuXIlr776Kqeddhqtra10dnZyySWX0NXVxYknnsjll1/O6aefzvPPP88pp5zCv//7v8dwFOMn7ruklgDnAi+Z2QvhvCuA/QHc/SbgTOCvzKwPeAv4qI9zmx1mpvakRKIe+iy8+VLcUQxv7rvh5C/t8WbOO+88rrjiCu644w4uvvjiActuu+228joAs2fP5re//S377rvvgPW2bdvGkiVL+PSnP80555zDtGnT9jiuySDuu6Secndz9yPcfVH4etDdbwqTBe5+g7u/y90Xuvsx7v7jiYgtl1UT5yJJdO6555JKpcrJoaSnp4c777yT9vZ2Tj75ZACy2eygZAEwa9YsLrjgArZs2cJPf/rTCYl7IsRdwpi01ImSSMQY/HJvFPvuuy/HH388jzzyCK+88goLFiwA4P7776ezs5PLLruMTKb/q/Pll1/my1/+Mk888QTr169n9+7dA7b3+uuvT2j840kJYwj5FnXTKpJU559/Po888gi33XYb1157LTD4chQEleTHHXccfX19HH/88Zx++unMnDmTVCrFCy+8wH333Ud3d3csxzAelDCGkMtm2LyzJ+4wRCQGZ5xxBjNnzuRb3/oWX/ziF9m8eTMPPfQQCxcuZOHCheX1rr76at566y2WL1/O0qVLB2zjmmuu4b777pvgyMdX7LfVTla5liZdkhJJqGnTpnHWWWfxxhtv8Oijj/Kd73yHvr6+AaULgDVr1tDa2jooWQA8/vjjExTtxFHCGIIqvUWS7fzzzwfg9ttv5/bbbyeTyXDOOecMWGf+/Pl0dnbys5/9bMD8W265hYcffniiQp0wuiQ1hFw2rRKGSIItWbKEgw8+mLvvvpve3l5OO+002tvbB6xz6aWX8vDDD3Psscdy1llnMWvWLFatWsVTTz3FmWeeyT333BNT9ONDJYwh5LJNvNVboK+gXvdEkuq8886jt7e3PF7ppJNO4v7772fBggXcdddd3HLLLWSzWZYvX86pp5460eGOO5uK/VYvXrzYV61atUfbuOWp3/CFB17hxc99kFnTm8YoMpHJa/Xq1Rx22GFxhyHjoNbP1syedffFQy1XCWMI+XIDhGoeREQElDCG1N/EeSHmSEREJgcljCH0d9OqEoaICChhDEndtIqIDKSEMYR8uYShhCEiAkoYQ8qpEyURkQGUMIaQUwlDEmgq3mafdGP5mSphDGFGs+owJFnS6XT5ITWZOnp7e0mn02OyLSWMIaRSFvSJoRKGJEQ+n2f79u1xhyFjbPv27eTz+THZlhLGMNSJkiRJa2srW7ZsYdOmTfT09OjyVANzd3p6eti0aRNbtmyhtbV1TLarxgeHkVMnSpIg2WyW/fffn87OTtauXUuhoIdWG1k6nSafz7P//vuTzWbHZJtKGMOYoSbOJWGy2Szz5s1j3rx5cYcik5AuSQ0jn82wY7cqAUVEQAljWKr0FhHpp4QxjFyLKr1FREqUMIahblpFRPopYQwj35JhZ3efbi8UEUEJY1i5bIaiw1u9ur1QREQJYxhqgFBEpJ8SxjBy5W5alTBERJQwhpFXCUNEpEwJYxi5bBOgJs5FREAJY1jlS1IqYYiIKGEMp3xJSiUMEREljOGUe91Te1IiIkoYw5mhblpFRMpiTRhmtp+ZLTezV8zsZTO7pMo6Zmb/YmZrzOxnZnbURMXXnEnRnEnptloREeLvD6MP+JS7P2dmeeBZM3vE3V+JrHMycEj4eg/w1XA4IfLqdU9EBIi5hOHu6939uXC8C1gN7FOx2oeA2z3wNLCXmU1Y7y7qdU9EJDBp6jDMbD5wJPBMxaJ9gN9Fpl9jcFLBzC40s1Vmtmrjxo31BeEOu7dBsb/tKPXrLSISGLOEYWbvNLPLzOwiM5s1yvfmgO8Bl7r79nr27+43u/tid1/c1tZWzybgpXvgS/tD56/Ls9TEuYhIYNQJw8w+Z2brzaw1Mu8E4HngK8CNwHNmtneN22siSBbfdvfvV1nldWC/yPS+4byxlwsTzY7fl2eVmjgXEUm6ekoYJwOvuntnZN41gAOfJ6iUPhAYdMdTJTMz4BZgtbtfN8RqPwD+Irxb6hhgm7uvryPukeU6gmEkYaibVhGRQD13Sc0H7i1NmNk+wB8A17n71eG8dwIfBj43wraWAOcCL5nZC+G8K4D9Adz9JuBB4BRgDbAL+HgdMdemnDA29M9SN60iIkB9CWM2EC1dLCEoXTwQmfcscNFIG3L3pwAbYR0HPjn6MOswbTakmqDrzfKsXLZJdRgiItR3SWojA+9S+mOgl4F3NzXXue14mQWljEgJI9+SoaevSHefet0TkWSrp4TxAnC6mR0O7AbOBp5y97ci68wHxqeeYbzlOwbVYQDs7C6QzaTjikpEJHb1lAL+AZgFvAj8Ihz/x9JCM0sTXKZaNRYBTrhc9YShegwRSbpRlzDc/Ukz+1PgfxDUXXzb3R+KrPI+gtte7632/kkv1w6v/bR/sqXUTatarBWRZKurLSl3/yHwwyGWPUnwxHZjynXAzk1Q6IN0hrxKGCIiwBhXTJvZbDObMZbbnHC5DsBhZ9C8SE6dKImIAPU96X28mf2Dmc2OzGs3s8eBTUCnmQ31EN7kV/HwnvrEEBEJ1FPC+GvgI+6+JTLvK8D7gV8Bm4FLzOysMYhv4lU8vJdXv94iIkB9CWMh8FRpwsymAWcCj7j7ocA7CFqX/csxiXCi5QeWMHRJSkQkUE/CaAfeiEy/B2gBboVyvxYPECSOxjOjPRjuCJ72ntaUJmWq9BYRqSdhdAPTItPvJ7i99onIvO1AK42oqQVaZpUvSZmZGiAUEaG+hPEb4LjI9J8B/+Xu0SbH9yOoAG9MFQ/v5VualDBEJPHqSRi3Ae82s2fM7Eng3cB3KtY5guAp8MaU64CuiibOdUlKRBKunoTxVeBOYDFBEyAPANeWFoZtTL0bWDEG8cWjsnkQ9estIlJX0yC9wH83s78MJr2rYpU3CZ70Xrvn4cWkosXaXDbD1rfUNIiIJFtdTYMADNX3trtvopHrLyC4tbZ3J3R3QTZPriXDa1t2xR2ViEis6k4YZjYd+AhBaWIvYBvwHHCvu+8ck+jiEn14L5snr7ukRETqSxhmdgpB5XcrA3vMc+CfzOzj7v5A1Tc3glzpWYzfw95vV6W3iAh1JAwzOwr4PpAGvg08RtBZ0jyC220/BtxjZkvc/dkxjHXi5OYGw8jT3jt7ChSKTjo1bI+yIiJTVj0ljL8lKEm8392frlh2q5n9G8EdUlcQPKPReEqXpMJba8u97vX0MbOlKa6oRERiVc9tte8H7q6SLABw92eAe8L1GtO02ZDKlEsY+Rb1iSEiUk/CmEXQuOBw1gEz69j25JBKBW1KhbfWqolzEZH6EsYbwNEjrLOYoF6jceU7yg0Q5tTEuYhIXQnjQeA4M/usmaWjC8wsZWafAk4I12tckae982riXESkrkrvLwAfBv4euChsT2o9MBc4FphP8LT31WMTYkxy7fDG88FoNqjoVh2GiCRZPU2DvGlmS4CvAScCB1Ss8gjwl+7e2JekcnODfr2LhUgnSmoeRESSq64H99x9LfAnZrYPwZPeswie9H6+opnzxpVrBy/Czk3kskHXHju6CzEHJSISn7qbBgEIk8PUSBCVcv1dteY6gie/dUlKRJJsxIRhZl+vc9vu7p+o873xy5ee9t5Aep4xvTmtS1Iikmi1lDDOr3PbDjRuwsgN7Ntb3bSKSNLVkjAOHPcoJqMZkQYICdqT0nMYIpJkIyYMd//tRAQy6TRPh+zM8tPeauJcRJKungf3kiPXAV3hJakWNXEuIsmmhDGcSFetqsMQkaSLNWGY2dfNbIOZ/XyI5UvNbJuZvRC+PjehAeba++swsk2qwxCRRIu7hHErcNII6zzp7ovC199NQEz98nMjJYy0ShgikmixJgx3fwLojDOGYeXaoacLenYGdRjdfbh73FGJiMQi7hJGLd5rZi+a2UNm9q6hVjKzC81slZmt2rhx49jsOfq0d7aJQtHZ3Vscm22LiDSYyZ4wngMOcPeFwL8C/zHUiu5+s7svdvfFbW1tY7P38sN7G8oNEHbpaW8RSahJnTDcfbu77wjHHwSazGzOhAWQC5sH6XqTfFbdtIpIsk3qhGFmc83MwvGjCeLdPGEBlC9JbSj3uqeKbxFJqj1qrXZPmdkdwFJgjpm9BnweaAJw95uAM4G/MrM+4C3goz6Rtc7T9wZLB3UYbUoYIpJssSYMd//YCMtvAG6YoHAGS6XCZzHe7C9h6JKUiCTUpL4kNSnk2mHHBvXrLSKJp4QxklxHeFutEoaIJJsSxkjC9qTKt9XqkpSIJJQSxkjChJFNQXM6pRKGiCSWEsZIch3gBdjVqSbORSTRlDBGkuvveU9NnItIkilhjCQfPu0d3lqrOgwRSSoljJFE25PKZtihtqREJKGUMEYSbbG2RZekRCS5lDBG0jwDmvPQFdZh6JKUiCSUEkYtwq5aVcIQkSRTwqhF+CxGXpXeIpJgShi1KJUwshm6+4r0FtTrnogkjxJGLfJzy5ekAHbqspSIJJASRi1y7dC9nVmZ4JZaXZYSkSRSwqhFeGvt3mwD1GKtiCSTEkYtwr699yp0AkoYIpJMShi1CJ/2nllKGLokJSIJpIRRi/CS1IzezQB0qYQhIgmkhFGLGXPAUkzvDhKGShgikkRKGLVIpWFGG9ndGwHUAKGIJJISRq1y7WTe2oCZShgikkxKGLXKdWA7NpBrzqgOQ0QSSQmjVrm5QYu16qZVRBJKCaNWuXbYuYF8c0rPYYhIIilh1CrXAcU+5mbfUsIQkURSwqhVPngWY5/0drUlJSKJpIRRq/DhvXnpbSphiEgiKWHUKkwY7bZNzZuLSCIpYdSq3GLtVt0lJSKJpIRRq2wOmmbQ6lvY0dNHsehxRyQiMqGUMEYj186sQifusKu3EHc0IiITSgljNHId5PvUxLmIJJMSxmjkO5jevQlQA4QikjyxJgwz+7qZbTCznw+x3MzsX8xsjZn9zMyOmugYB8h10BImDD2LISJJE3cJ41bgpGGWnwwcEr4uBL46ATENLddOU+92svToWQwRSZxYE4a7PwF0DrPKh4DbPfA0sJeZzZuY6KoI+/Zus22qwxCRxIm7hDGSfYDfRaZfC+cNYmYXmtkqM1u1cePG8YkmfBajja1q4lxEEmeyJ4yaufvN7r7Y3Re3tbWNz05y7QC0mR7eE5HkmewJ43Vgv8j0vuG8eJRKGKb2pEQkeSZ7wvgB8Bfh3VLHANvcfX1s0cxoA4y3qQFCEUmgTJw7N7M7gKXAHDN7Dfg80ATg7jcBDwKnAGuAXcDH44k0lM7AjDnM27Wd13RJSkQSJtaE4e4fG2G5A5+coHBqk+ugY7dKGCKSPJP9ktTkk+ugja1q4lxEEkcJY7RyHbSyRXdJiUjiKGGMVq6dvQpb6NqttqREJFmUMEYrP5cMfaS6h3tAXURk6lHCGK3w4b2W3ZtjDkREZGIpYYxW+PDejN7NBDdxiYgkgxLGaIUJY3ZxC919xZiDERGZOEoYoxUmjHbbqmcxRCRRlDBGK5unL92iJs5FJHGUMEbLjJ6WOUGLtSphiEiCKGHUoW96O+1sVTetIpIoShh18BkdKmGISOIoYdTB8u1hnxh62ltEkiPW1mobVXrmPGbaDnbt2hV3KCIiE0YljDo0z5oLQKHr9zFHIiIycZQw6pAJE4bt2BBzJCIiE0cJow4WPryX3qWEISLJoYRRj3xQwti64TWeW7eFYlFtSonI1KdK73rMaAOgd+t6PnLjj2nLZznhsA4+uKCD9759b1qa0jEHKCIy9pQw6pFugul788k5G3hv+5v8eD2seOHXPLpyOj3Ne7HkHfM4cUEHx72jg1nTm+KOVkRkTChh1GveQjK/eoyj1z3J0cClKaAlWNT1X9PZ/Is8vyaPT2slt1cbzc1NpFMpMuk06XSKTDpFJp0Jhpk0mXSalKWCZNQ0DTIt1YdN0yAzDTJZSGUglQ6GlqqYTofj6f5xS0XGLc6zJyINSAmjXufcAzs3wa7Nkdcm2NVJbucmeje9iW9eT1/XJlrW/wYzJ0XQHLpTpA8oUKQHMJxU+Gq2PrL0kGJ860U83GvRUrilcNK4GY4BFiYUC9clMm14mGwMB/dgdS+vGcwfEH8KtyBJeZi03CwcpsJEFsRR2gflfRLuLxJXdH0rJcTIdDQ5AngR8yJQDMc9HBaC+AnmDXp/NNmmgmWWSgdxlPtCCc9BeEZL4/3ngmBbgFmqfB4tFRk3oz+BRxJ5+TxHz0f0HARDK523yDZJpYJjLBag2IcX+oLjLfaFr0L46gMv9O97wD4s2HflfsvnJdP/o6Q0Xvo8zLBiLxR6oa8bCj3BeKE7HPb0zzPDUk2QzkCqCdLNkfHwVRpn4LFTOr/l6cjyqirne3AePDwfXqyYLkCx2H+OyvuwKvuMnP8B4xXnMLq89Pfikf+ZAfOI/K1VHsoQx3jgH8GhfzLE8e8ZJYx6pdKQ7wheFQxoDV8AnTt72LG7jx29fezsLrCrp49dPZFhd4Gd4XhPX5FCoYgXurG+3aT6ukkVdpMu7Mb63iJTDKeL3VAsYMUi7n3BF4MXsGJf8OXo4bDYRyr8srTwS9K8iJWmiwVS0ekBX/rB0MrD/nGPJJP+caP/azRYuz8ZFgcM0xbsK00xnBfM7//a7N8fFfsOtlMgRS9pnJQF709HtlMaJziy8BXsoTReHnqwl1SY1NORbUWH/dtkQHTRaa8Yr3buSmcpiGDguY7+DQGYDfyyiL4nFdlTaX4wDJYVSdFHigJp+khTIEWfB8NCeX5wxirPeXRIxblPUwjOh/Wfl/K88GU4vWToIRMMPRiW5vWQodcz9BEk9SZ2krE+miiQoUAzfWSsEE73lYfVjr3auRj0Dxk5p5WK4fkYaVg6R6kq+4z+6CMyDQM/94HnuDjgL6L8Yw2I/h1B//9Y/yEN/lspWb2+m6OUMBpX64xmWmc0xx1GzdydokPRnaI7Hv7YKU0Xi+CE86F/HfrXdcJthHeQFSPbjG6/UAzeUwjX83D//ePlqMrjRQ/WKe274NBb2nY4j3CbtV55Kx1LKbbo9svTkTgHvX+I8zhwm/0xlrZX2n7l+4aP0wfEXG1+NdFzYVW+Pod6f/nzGGK71Y+9+rrV9lk+jvK56f/7KR+XV9/PkNstn9/o36WXp6OfaXBeLCgI0H+ebMA8i3ye0c+xfzvRmK0iSUW3GZ0unatSjME5GXjc/fO8YnnkWPtPKCcu6OCoUZyr0VDCkEHMjLRBesjfZCKSRHoOQ0REaqKEISIiNVHCEBGRmihhiIhITZQwRESkJkoYIiJSEyUMERGpiRKGiIjUxIZ7qrRRmdlG4Ld1vn0OsGkMw5kMptoxTbXjgal3TFPteGDqHVO14znA3duGesOUTBh7wsxWufviuOMYS1PtmKba8cDUO6apdjww9Y6pnuPRJSkREamJEoaIiNRECWOwm+MOYBxMtWOaascDU++YptrxwNQ7plEfj+owRESkJiphiIhITZQwRESkJkoYEWZ2kpn9wszWmNln445nLJjZWjN7ycxeMLNVccczWmb2dTPbYGY/j8xrNbNHzOy/wuHsOGMcrSGOaZmZvR5+Ti+Y2SlxxjgaZrafmS03s1fM7GUzuySc35Cf0zDH08ifUYuZrTSzF8Njuiqcf6CZPRN+591lZsN2Dao6jJCZpYFfAicCrwE/BT7m7q/EGtgeMrO1wGJ3b8gHjszsj4AdwO3ufng47x+ATnf/UpjYZ7v7Z+KMczSGOKZlwA53/0qcsdXDzOYB89z9OTPLA88CHwbOpwE/p2GO5ywa9zMyYIa77zCzJuAp4BLgcuD77n6nmd0EvOjuXx1qOyph9DsaWOPuv3b3HuBO4EMxx5R47v4E0Fkx+0PAbeH4bQT/zA1jiGNqWO6+3t2fC8e7gNXAPjTo5zTM8TQsD+wIJ5vClwPHAfeE80f8jJQw+u0D/C4y/RoN/kcScuBHZvasmV0YdzBjpMPd14fjbwIdcQYzhi42s5+Fl6wa4vJNJTObDxwJPMMU+Jwqjgca+DMys7SZvQBsAB4BfgVsdfe+cJURv/OUMKa+Y939KOBk4JPh5ZApw4NrqlPhuupXgbcDi4D1wD/GGk0dzCwHfA+41N23R5c14udU5Xga+jNy94K7LwL2Jbii8s7RbkMJo9/rwH6R6X3DeQ3N3V8PhxuAewn+UBrd78PrzKXrzRtijmePufvvw3/oIvB/aLDPKbwu/j3g2+7+/XB2w35O1Y6n0T+jEnffCiwH3gvsZWaZcNGI33lKGP1+ChwS3jXQDHwU+EHMMe0RM5sRVtphZjOADwI/H/5dDeEHwHnh+HnAfTHGMiZKX6yhM2igzymsUL0FWO3u10UWNeTnNNTxNPhn1GZme4Xj0whu7llNkDjODFcb8TPSXVIR4W1y/wykga+7+9/HG9GeMbODCEoVABngO412TGZ2B7CUoCnm3wOfB/4D+C6wP0Ez9me5e8NUIg9xTEsJLnU4sBa4KHL9f1Izs2OBJ4GXgGI4+wqC6/4N9zkNczwfo3E/oyMIKrXTBAWF77r734XfEXcCrcDzwJ+7e/eQ21HCEBGRWuiSlIiI1EQJQ0REaqKEISIiNVHCEBGRmihhiIhITZQwRBqUma0wM93mKBNGCUNERGqihCEiIjVRwhARkZooYUjimdl7zOweM3vTzHrM7Hdm9jUze1vFeivMzM0sa2ZXm9lvzKzbzH5lZp8fqrcyMzvezH5oZp3h+r80sy+Z2awh1m81s783s5+b2S4z2xb2lPalsE2wyvUzZnZF2LNddxj/tSP1niYyWmoaRBLNzC4Abga6CRrL+x1wCHA6QTtPx7j7unDdFcAHwvX+kKDjmV6CjoLeDjwAnO6Rfyozu4igWeydwN0ELbYuBd4DvAIsCVsPLa1/IEGDcAcQ9PT2OMEPu0OBE4B3uPvainjuBt4PPARsB04Jj+FWd//4WJwnEQDcXS+9Evki+BLuAdYA+1QsOx4oAPdG5q0gaHjulwTdjZbmtwA/CZedG5l/AEEi2g68s2L7N4br31wx/8fh/L+pEu8coKVKPM8CrZH5M8JjKgBz4z7Pek2dly5JSZL9FUFXlZd42G9Iibv/P4KSxGmlJuIjvuDuWyLr7gb+Jpy8ILLenwPNwA3u/mrFNv4W6ALONbMsgJn9AUEfBS8A11YG6+6bwn1V+oxHWoF1953AtwlKJourrC9Sl8zIq4hMWe8Nhx8wsz+ssrydoDnoQwl+xZc8XmXdpwh+0R8ZmXdUOHyscmV332JmzwN/RNDz2YvAMeHihz3opKdWq6rMK3U33FDdiMrkpoQhSbZ3OPzfI6yXq5j+feUK7t5nZpsIkkxJqVJ7qD4TSvP3qhiOqqdHj9SBRJT6aU6PZlsiw1HCkCTbFg5neUUf1CPoANZFZ4TdXM4hqK+o3P5c4OUq25lXsd7WcLjPKGIRmTCqw5Akezocvn+U7/tAlXnHEvyafz4yrzS+tHLlsLvMRcBugq4yo/H8iZnpf1MmHf1RSpLdQHBb7D+Z2aGVC82s2cyqJZMrzWx2ZL0W4Jpw8huR9b4Vbv+vzezgim18AZgJfMvDLjHd/VmCu6QWAZ+pEs/e4b5EYqFLUpJY7v5q+BzG14GXzeyHBLfMNhH0Q/1+YCNBpXTU6nD9yucw/i/wzcj215rZpcC/Ac+Z2XfD7X2AoML9VQYnhj8nuF32i2b2Z+G4ETxX8cEwlrV7fPAidVDCkERz92+Z2YvAp4A/JvhS3gm8QfBg3l1V3nYWcCVwDvA2gkrqZcCX3H3Ak7DufqOZrQH+F/BnwHSCO5i+DHyxssLa3X9jZkcBnwY+DFxMcNlqLfCPBA/+icRCT3qL1Kj0ZLW7W9yxiMRBdRgiIlITJQwREamJEoaIiNREdRgiIlITlTBERKQmShgiIlITJQwREamJEoaIiNRECUNERGry/wFyXYY8TO+pdwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_loss_list, label='train')\n",
    "plt.plot(val_loss_list, label='val')\n",
    "plt.legend(fontsize=20)\n",
    "plt.xlabel('epoch',fontsize=20)\n",
    "plt.ylabel('loss', fontsize=20)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b3cb7c3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}