scripts/learning-pipeline/plotting_heuristics_of_ML_pipeline_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ef885fee",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-17T09:18:05.695513Z",
     "start_time": "2021-12-17T09:18:03.125501Z"
    }
   },
   "source": [
    "# Plotting heuristics of Machine Learning pipeline\n",
    "\n",
    "**Author**: Miguel Xochicale [@mxochicale](https://github.com/mxochicale)     \n",
    "**Contributors**: Nhat Phung Tran Huy [@huynhatd13](https://github.com/huynhatd13); Hamideh Kerdegari [@hamidehkerdegari](https://github.com/hamidehkerdegari);  Alberto Gomez [@gomezalberto](https://github.com/)  \n",
    "\n",
    "\n",
    "## History\n",
    "* June2022: Adding multiple plots  \n",
    "* Aug2022: Ploting multiple trains\n",
    "\n",
    "## Summary\n",
    "This notebook presents a learning pipeline to classify 4 chamber view from echocardiography datasets.\n",
    "\n",
    "### How to run the notebook\n",
    "\n",
    "1. Go to echocardiography repository path: `$HOME/repositories/echocardiography/`\n",
    "2. Open echocardiography repo in pycharm and in the terminal type:\n",
    "    ```\n",
    "    git checkout master # or the branch\n",
    "    git pull # to bring a local branch up-to-date with its remote version\n",
    "    ```\n",
    "3. Launch Notebook server  \n",
    "    Go to you repository path: `cd $HOME/repositories/echocardiography/scripts/dataloaders` and type in the pycharm terminal:\n",
    "    ```\n",
    "    conda activate rt-ai-echo-VE \n",
    "    jupyter notebook\n",
    "    ```\n",
    "    which will open your web-browser.\n",
    "    \n",
    "    \n",
    "### References\n",
    "* \"Proposed Regulatory Framework for Modifications to Artificial Intelligence/Machine Learning (AI/ML)-Based Software as a Medical Device (SaMD) - Discussion Paper and Request for Feedback\". https://www.fda.gov/media/122535/download \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e93f2aa0",
   "metadata": {},
   "source": [
    "# Jupyter Notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "785d3efa",
   "metadata": {},
   "source": [
    "## 1. Setting imports and datasets paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f345efe8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-14T23:47:30.234885Z",
     "start_time": "2022-08-14T23:47:29.731501Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'N_CLIPS_BATCHES' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_36917/2919148599.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[0mRESULTS_PATH\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'scripts/learning-pipeline/results/aug-09-2022_SqueezeNet_hyperparameters/frames_per_clip/'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mNUMBER_OF_SUBJECTS\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mN_CLIPS_BATCHES\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'N_CLIPS_BATCHES' is not defined"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import fnmatch\n",
    "import sys\n",
    "import argparse\n",
    "import yaml\n",
    "from pathlib import Path\n",
    "import json    \n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.gridspec as gridspec\n",
    "import matplotlib.animation as animation\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "\n",
    "from typing import Dict, List, Tuple\n",
    "\n",
    "from source.helpers.learning_pipeline import json2DataFrame, \\\n",
    "                                            jsonPARAMS2DataFrame\n",
    "\n",
    "\n",
    "HOME_PATH = os.path.expanduser(f'~')\n",
    "USERNAME = os.path.split(HOME_PATH)[1]\n",
    "\n",
    "REPOSITORY_PATH='repositories/echocardiography'\n",
    "FULL_REPO_PATH = HOME_PATH+'/'+REPOSITORY_PATH\n",
    "\n",
    "#### Setting RESULTS_PATH and TRAINING_CURVES_PATH\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/1st-tests-23-june-2022'\n",
    "#RESULTS_PATH='scripts/learning-pipeline/results/3rd-tests-30-june-2022/experiments-01-02-03-04'\n",
    "#RESULTS_PATH='scripts/learning-pipeline/results/3rd-tests-30-june-2022/experiments-04-05-06-07'\n",
    "#RESULTS_PATH='scripts/learning-pipeline/results/3rd-tests-30-june-2022/experiments-09-10-11-12'\n",
    "\n",
    "\n",
    "# ####################################\n",
    "# ##MobileNetV1() with 3,208,450 params\n",
    "# # NUMBER_OF_SUBJECTS='05-subjects'\n",
    "# NUMBER_OF_SUBJECTS='31-subjects'\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/mobileNetV1-8-aug-2022/'+NUMBER_OF_SUBJECTS\n",
    "\n",
    "#######################################\n",
    "##MobileNet2() with 2,225,858 params\n",
    "# NUMBER_OF_SUBJECTS='05-subjects'\n",
    "# # NUMBER_OF_SUBJECTS='31-subjects'\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/5th-tests-04-augs-2022/'+NUMBER_OF_SUBJECTS\n",
    "\n",
    "\n",
    "# ####################################\n",
    "# ##SqueezeNet_source0() with 733,580 params\n",
    "# # NUMBER_OF_SUBJECTS='05-subjects'\n",
    "# NUMBER_OF_SUBJECTS='31-subjects'\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/6th-tests-05-aug-2022/'+NUMBER_OF_SUBJECTS\n",
    "\n",
    "####################################\n",
    "##SqueezeNet_source0() with 733,580 params\n",
    "\n",
    "# NUMBER_OF_SUBJECTS='05-subjects'\n",
    "# # BATCHES='-batches_05'\n",
    "# # BATCHES='-batches_10'\n",
    "# BATCHES='-batches_30'\n",
    "\n",
    "# NUMBER_OF_SUBJECTS='31-subjects'\n",
    "# # BATCHES='-batches_05'\n",
    "# # BATCHES='-batches_10'\n",
    "# BATCHES='-batches_30'\n",
    "\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/aug-09-2022_SqueezeNet_hyperparameters/batches/'+ \\\n",
    "#         NUMBER_OF_SUBJECTS+BATCHES\n",
    "\n",
    "\n",
    "# NUMBER_OF_SUBJECTS='05-subjects'\n",
    "# BATCHES='-batches_30'\n",
    "# LR='0_01'\n",
    "# # LR='0_0001'\n",
    "# # LR='0_00001'\n",
    "# RESULTS_PATH='scripts/learning-pipeline/results/aug-09-2022_SqueezeNet_hyperparameters/learning_rate/'+ \\\n",
    "#         NUMBER_OF_SUBJECTS+BATCHES+'/lr_'+LR\n",
    "\n",
    "\n",
    "\n",
    "####################################\n",
    "##SqueezeNet_source0() with 733,580 params\n",
    "\n",
    "###########################\n",
    "#### FRAMES PER CLIP\n",
    "\n",
    "NUMBER_OF_SUBJECTS='05-subjects'\n",
    "\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_01___BATCH_SIZE_CLIPS_30'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 1\n",
    "# BatchClips=30\n",
    "# LR=0.001\n",
    "\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_02___BATCH_SIZE_CLIPS_30'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 2\n",
    "# BatchClips=30\n",
    "# LR=0.001\n",
    "\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_05___BATCH_SIZE_CLIPS_30'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 5\n",
    "# BatchClips=30\n",
    "# LR=0.001\n",
    "\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_10___BATCH_SIZE_CLIPS_30'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 10\n",
    "# BatchClips=30\n",
    "# LR=0.001\n",
    "\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_20___BATCH_SIZE_CLIPS_30'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 20\n",
    "# BatchClips=30\n",
    "# LR=0.001\n",
    "\n",
    "# N_CLIPS_BATCHES='NUMBER_OF_FRAMES_PER_SEGMENT_IN_A_CLIP_30___BATCH_SIZE_CLIPS_60'\n",
    "# FRAMES_PER_SEGMENT_IN_A_CLIP = 30\n",
    "# BatchClips=60\n",
    "# LR=0.001\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "RESULTS_PATH='scripts/learning-pipeline/results/aug-09-2022_SqueezeNet_hyperparameters/frames_per_clip/'+ \\\n",
    "        NUMBER_OF_SUBJECTS + '/' + N_CLIPS_BATCHES\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###################\n",
    "## LEARNING RATES\n",
    "# # LR='0_01'\n",
    "# # LR='0_0001'\n",
    "# # LR='0_00001'\n",
    "# LR='0_01'\n",
    "# LR='0_0001'\n",
    "# LR='0_00001'\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#####################\n",
    "##Ploting settings\n",
    "TRAINING_CURVES_PATH = os.path.join(FULL_REPO_PATH, RESULTS_PATH)\n",
    "PLOTTING_RESULTS_PATH = TRAINING_CURVES_PATH + '/plotting_results/'\n",
    "os.makedirs(PLOTTING_RESULTS_PATH, exist_ok=True)\n",
    "DPI_val=200\n",
    "\n",
    "\n",
    "## Printing Versions and paths\n",
    "print(f'Pandas Version: {pd.__version__}')\n",
    "print(f'seaborn Version: {sns.__version__}')\n",
    "print(f'TRAINING_CURVES_PATH: {TRAINING_CURVES_PATH}' )\n",
    "print(f'PLOTTING_RESULTS_PATH: {PLOTTING_RESULTS_PATH}' )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bf5d1cd",
   "metadata": {},
   "source": [
    "## 2. Reading dictionaries and filtering files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20bd99e0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-14T23:47:30.235944Z",
     "start_time": "2022-08-14T23:47:30.235936Z"
    }
   },
   "outputs": [],
   "source": [
    "print(f'TRAINING_CURVES_PATH: {TRAINING_CURVES_PATH}' )\n",
    "os.chdir(TRAINING_CURVES_PATH)\n",
    "\n",
    "pattern = 'TEMP_DICT_TRAINING_CURVES_FOR_____LOSS_ACC*.json'\n",
    "json_files=fnmatch.filter(  sorted(os.listdir(TRAINING_CURVES_PATH), key=lambda x: x[-6])  , pattern)\n",
    "print(f'-------------------')\n",
    "print(f'-----Stats')\n",
    "for i in range(0,len(json_files)):\n",
    "    print(i, json_files[i])\n",
    "    \n",
    "    \n",
    "pattern_params = 'TEMP_DICT_TRAINING_CURVES_FOR____TRAINING_PARAMETERS*.json'\n",
    "json_files_params=fnmatch.filter(  sorted(os.listdir(TRAINING_CURVES_PATH), key=lambda x: x[-6])  , pattern_params)    \n",
    "print(f'-------------------')\n",
    "print(f'----Parameters')\n",
    "for i in range(0,len(json_files_params)):\n",
    "    print(i, json_files_params[i])    \n",
    "\n",
    "\n",
    "RUN_VERSION=[json_files[0][-12:-5], json_files[1][-12:-5], json_files[2][-12:-5]]\n",
    "print(f'-------------------')\n",
    "print(f'----RUN_VERSION')\n",
    "for i in range(0,len(RUN_VERSION)):\n",
    "    print(i, RUN_VERSION[i])    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8bfaa80",
   "metadata": {},
   "source": [
    "## 2. Plotting training curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79f90692",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-14T23:47:30.238197Z",
     "start_time": "2022-08-14T23:47:30.238187Z"
    }
   },
   "outputs": [],
   "source": [
    "TYPE_str = 'ACC'    \n",
    "STR_VARIABLE_NAME = 'Acc [%]'\n",
    "# RUN_VERSION=['run02', 'run00', 'run01']\n",
    "\n",
    "acc0_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[0]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[0])\n",
    "acc1_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[1]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[1])\n",
    "acc2_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[2]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[2])\n",
    "\n",
    "\n",
    "acc_dfall = [acc0_all_, acc1_all_,acc2_all_]\n",
    "acc_all = pd.concat(acc_dfall)\n",
    "print(acc_all)\n",
    "\n",
    "\n",
    "sns.set(style=\"darkgrid\")#sns.set(style=\"ticks\")\n",
    "g0=sns.relplot(x=\"epochs\", y=STR_VARIABLE_NAME, \n",
    "            hue=\"RUN_NN\",\n",
    "            col=\"N_BatchClips\", \n",
    "            row=\"FRXClips\", \n",
    "            style=\"datatype\",\n",
    "            #col_wrap=2, \n",
    "            height=3, aspect=1.75, linewidth=3.5,\n",
    "            kind=\"line\", \n",
    "            estimator=None,\n",
    "            palette=\"Set2\",#palette=palette,\n",
    "            data=acc_all.query(\"datatype == 'train'\"))#data=acc_all)\n",
    "g0.set(ylim=(0, 115))\n",
    "\n",
    "sns.set(style=\"darkgrid\")\n",
    "g1=sns.relplot(x=\"epochs\", y=STR_VARIABLE_NAME, \n",
    "            hue=\"RUN_NN\",\n",
    "            col=\"N_BatchClips\", \n",
    "            row=\"FRXClips\", \n",
    "            style=\"datatype\",\n",
    "            #col_wrap=2, \n",
    "            height=3, aspect=1.75, linewidth=3.5,\n",
    "            kind=\"line\", \n",
    "            estimator=None,\n",
    "            palette=\"inferno\",#palette=palette,\n",
    "            data=acc_all.query(\"datatype == 'test'\"))#data=acc_all)\n",
    "g1.set(ylim=(0, 115))\n",
    "\n",
    "\n",
    "g0.savefig(PLOTTING_RESULTS_PATH+'ACC_train.png', dpi=DPI_val)\n",
    "g1.savefig(PLOTTING_RESULTS_PATH+'ACC_test.png', dpi=DPI_val)\n",
    "plt.show()\n",
    "\n",
    "\n",
    "TYPE_str = 'LOS'    \n",
    "STR_VARIABLE_NAME = 'Loss'\n",
    "loss0_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[0]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[0])\n",
    "loss1_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[1]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[1])\n",
    "loss2_all_=json2DataFrame(TRAINING_CURVES_PATH,str(json_files[2]), \n",
    "                    TYPE_str, FRAMES_PER_SEGMENT_IN_A_CLIP, BatchClips,LR, STR_VARIABLE_NAME, RUN_VERSION[2])\n",
    "\n",
    "loss_dfall = [loss0_all_, loss1_all_,loss2_all_]\n",
    "loss_all = pd.concat(loss_dfall)\n",
    "print(loss_all)\n",
    "\n",
    "\n",
    "sns.set(style=\"darkgrid\")#sns.set(style=\"ticks\")\n",
    "g2=sns.relplot(x=\"epochs\", y=STR_VARIABLE_NAME, \n",
    "            hue=\"RUN_NN\",\n",
    "            col=\"N_BatchClips\", \n",
    "            row=\"FRXClips\", \n",
    "            style=\"datatype\",\n",
    "            #col_wrap=2, \n",
    "            height=3, aspect=1.75, linewidth=3.5,\n",
    "            kind=\"line\", \n",
    "            estimator=None,\n",
    "            palette=\"Set2\",#palette=palette,\n",
    "            data=loss_all.query(\"datatype == 'train'\"))#data=acc_all)\n",
    "g2.set(ylim=(-0.4, 2))\n",
    "\n",
    "sns.set(style=\"darkgrid\")\n",
    "g3=sns.relplot(x=\"epochs\", y=STR_VARIABLE_NAME, \n",
    "            hue=\"RUN_NN\",\n",
    "            col=\"N_BatchClips\", \n",
    "            row=\"FRXClips\", \n",
    "            style=\"datatype\",\n",
    "            #col_wrap=2, \n",
    "            height=3, aspect=1.75, linewidth=3.5,\n",
    "            kind=\"line\", \n",
    "            estimator=None,\n",
    "            palette=\"inferno\",#palette=palette,\n",
    "            data=loss_all.query(\"datatype == 'test'\"))#data=acc_all)\n",
    "g3.set(ylim=(-0.4, 2))\n",
    "\n",
    "g2.savefig(PLOTTING_RESULTS_PATH+'LOSS_train.png', dpi=DPI_val)\n",
    "g3.savefig(PLOTTING_RESULTS_PATH+'LOSS_test.png', dpi=DPI_val)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cde5d085",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-07-05T14:03:08.674646Z",
     "start_time": "2022-07-05T14:03:08.668357Z"
    }
   },
   "source": [
    "## 3. Plotting Paramters of Performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d09e1bca",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-14T23:47:30.238936Z",
     "start_time": "2022-08-14T23:47:30.238926Z"
    }
   },
   "outputs": [],
   "source": [
    "elapsed_time_for_the_NOTEBOOK_in_secs=[]\n",
    "elapsed_time_for_the_training_loop_in_secs=[]\n",
    "Precision=[]\n",
    "Recall=[]\n",
    "F1score=[]\n",
    "\n",
    "\n",
    "for i in range(0, len(json_files_params)):\n",
    "    params=jsonPARAMS2DataFrame(TRAINING_CURVES_PATH,str(json_files_params[i]))\n",
    "    elapsed_time_for_the_NOTEBOOK_in_secs.append(params[0])\n",
    "    elapsed_time_for_the_training_loop_in_secs.append(params[1])\n",
    "    Precision.append(params[11]['Precision']['weighted avg/Total'])\n",
    "    Recall.append(params[11]['Recall']['weighted avg/Total'])\n",
    "    F1score.append(params[11]['F1-score']['weighted avg/Total'])\n",
    "    \n",
    "print(f'elapsed_time_for_the_NOTEBOOK_in_secs {elapsed_time_for_the_NOTEBOOK_in_secs}')    \n",
    "print(f'elapsed_time_for_the_training_loop_in_secs {elapsed_time_for_the_training_loop_in_secs}')\n",
    "print(f'Precision {Precision}')   \n",
    "print(f'Recall {Recall}')   \n",
    "print(f'F1score {F1score}')\n",
    "\n",
    "\n",
    "##\n",
    "Y_AXIS_LABEL='Classification [%]'\n",
    "precision_report_df = pd.DataFrame(Precision).reset_index(drop=True)\n",
    "precision_report_df.rename(columns={0:Y_AXIS_LABEL}, inplace=True)\n",
    "precision_report_df.insert(0, 'Classification type', 'Precision', True)\n",
    "precision_report_df['RUN_NN'] = RUN_VERSION\n",
    "precision_report_df.insert(0, 'Dataset', NUMBER_OF_SUBJECTS, True)\n",
    "\n",
    "recall_report_df = pd.DataFrame(Recall).reset_index(drop=True)\n",
    "recall_report_df.rename(columns={0:Y_AXIS_LABEL}, inplace=True)\n",
    "recall_report_df.insert(0, 'Classification type', 'Recall', True)\n",
    "recall_report_df['RUN_NN'] = RUN_VERSION\n",
    "recall_report_df.insert(0, 'Dataset', NUMBER_OF_SUBJECTS, True)\n",
    "\n",
    "\n",
    "f1score_report_df = pd.DataFrame(F1score).reset_index(drop=True)\n",
    "f1score_report_df.rename(columns={0:Y_AXIS_LABEL}, inplace=True)\n",
    "f1score_report_df.insert(0, 'Classification type', 'F1score', True)\n",
    "f1score_report_df['RUN_NN'] = RUN_VERSION\n",
    "f1score_report_df.insert(0, 'Dataset', NUMBER_OF_SUBJECTS, True)\n",
    "\n",
    "classification_df = pd.concat([precision_report_df, recall_report_df, f1score_report_df])\n",
    "print(classification_df)\n",
    "g4=sns.barplot(x=\"RUN_NN\", y=Y_AXIS_LABEL, hue=\"Classification type\", \n",
    "                 data=classification_df)\n",
    "g4.set(ylim=(-0.01, 1.01))\n",
    "\n",
    "plt.savefig(PLOTTING_RESULTS_PATH+'df_classification.png', dpi=DPI_val)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "Y_AXIS_LABEL='Elapse_time_secs'\n",
    "timenotebook_df = pd.DataFrame(elapsed_time_for_the_NOTEBOOK_in_secs).reset_index(drop=True)\n",
    "timenotebook_df.rename(columns={0:Y_AXIS_LABEL}, inplace=True)\n",
    "timenotebook_df['RUN_NN'] = RUN_VERSION\n",
    "timenotebook_df.insert(0, 'Dataset', NUMBER_OF_SUBJECTS, True)\n",
    "timenotebook_df.insert(3, 'TimeFor', 'Notebook', True)\n",
    "\n",
    "timetraining_df = pd.DataFrame(elapsed_time_for_the_training_loop_in_secs).reset_index(drop=True)\n",
    "timetraining_df.rename(columns={0:Y_AXIS_LABEL}, inplace=True)\n",
    "timetraining_df['RUN_NN'] = RUN_VERSION\n",
    "timetraining_df.insert(0, 'Dataset', NUMBER_OF_SUBJECTS, True)\n",
    "timetraining_df.insert(3, 'TimeFor', 'Training', True)\n",
    "\n",
    "df_all = pd.concat([timenotebook_df, timetraining_df])\n",
    "\n",
    "g5=sns.barplot(x=\"RUN_NN\", y=Y_AXIS_LABEL, hue=\"TimeFor\", \n",
    "                 data=df_all)\n",
    "\n",
    "\n",
    "plt.savefig(PLOTTING_RESULTS_PATH+'df_timetraining.png', dpi=DPI_val)\n",
    "\n",
    "plt.show()\n",
    "\n",
    "\n",
    "##https://seaborn.pydata.org/generated/seaborn.barplot.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "536a20e0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21956b27",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2465ae07",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}