SparseChem_Train_cb29-sub_tasks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "89c58a9e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-03-04T18:50:52.221512Z",
     "start_time": "2022-03-04T18:50:52.203972Z"
    }
   },
   "source": [
    "## Train SparseChem on Chembl_mini \n",
    "Output to `experiments/SparseChem`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "baa58a9a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:36.231754Z",
     "start_time": "2022-11-19T21:27:36.200566Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:95% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:95% !important; }</style>\"))\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "55155d2e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.642544Z",
     "start_time": "2022-11-19T21:27:36.237162Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Pytorch thread count: 4\n",
      " Set Pytorch thread count to : 2\n",
      " Pytorch thread count set to : 2\n"
     ]
    }
   ],
   "source": [
    "# Copyright (c) 2020 KU Leuven\n",
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import os.path\n",
    "import time\n",
    "import json\n",
    "import functools\n",
    "import types\n",
    "import wandb\n",
    "from datetime import datetime\n",
    "import pprint\n",
    "import csv\n",
    "import copy \n",
    "from contextlib import redirect_stdout\n",
    "import sparsechem as sc\n",
    "from sparsechem import Nothing\n",
    "from sparsechem.notebook_modules import (check_for_improvement, init_wandb, initialize,\n",
    "                                        assertions, display_gpu_info)\n",
    "import scipy.io\n",
    "import scipy.sparse\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.optim.lr_scheduler import MultiStepLR\n",
    "from torch.utils.tensorboard import SummaryWriter\n",
    "from pytorch_memlab import MemReporter\n",
    "from pynvml import *\n",
    "\n",
    "pp = pprint.PrettyPrinter(indent=4)\n",
    "np.set_printoptions(edgeitems=3, infstr='inf', linewidth=150, nanstr='nan')\n",
    "torch.set_printoptions( linewidth=132)\n",
    "os.environ['WANDB_NOTEBOOK_NAME'] = 'SparseChem_Train_mini.ipynb'\n",
    "\n",
    "pytorch_threads = 2\n",
    "print(f\" Pytorch thread count: {torch.get_num_threads()}\")\n",
    "print(f\" Set Pytorch thread count to : {pytorch_threads}\")\n",
    "torch.set_num_threads(pytorch_threads)\n",
    "print(f\" Pytorch thread count set to : {torch.get_num_threads()}\")\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    nvmlInit()\n",
    "\n",
    "#import warnings\n",
    "# from torch.serialization import SourceChangeWarning \n",
    "#warnings.filterwarnings(\"ignore\", category=UserWarning)    \n",
    "\n",
    "# import multiprocessing\n",
    "# multiprocessing.set_start_method('fork', force=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f690d55",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2d8dd7f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.681679Z",
     "start_time": "2022-11-19T21:27:38.646600Z"
    }
   },
   "outputs": [],
   "source": [
    "# disp_gpu_device_info()\n",
    "\n",
    "# display_gpu_info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4db6bd47",
   "metadata": {},
   "source": [
    "### Setup command line parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "58c1586e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.716879Z",
     "start_time": "2022-11-19T21:27:38.688923Z"
    }
   },
   "outputs": [],
   "source": [
    "datadir=\"../MLDatasets/chembl29_10task\"\n",
    "outdir =\"../experiments/cb29-SparseChem-TaskGroups\"\n",
    "project_name = \"SC-cb29-TaskGroups\"\n",
    "x_file = \"chembl_29_X.npy\"\n",
    "# y_file = \"chembl_29_Y_all.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_0_cols_472.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_1_cols_624.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_6_cols_688.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_10_cols_192.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_11_cols_620.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_643_cols_184.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_836_cols_224.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_1005_cols_148.npy\"\n",
    "# y_file = \"chembl_29_Y_tg_1028_cols_344.npy\"\n",
    "y_file = \"chembl_29_Y_tg_1031_cols_72.npy\"\n",
    "folding_file = \"chembl_29_folding.npy\"\n",
    "dev = \"cuda:0\"\n",
    "batch_size = 4096\n",
    "\n",
    "cmd = (\n",
    "  f\" --project_name           {project_name} \" +\n",
    "  f\" --data_dir                    {datadir} \" +\n",
    "  f\" --output_dir                   {outdir} \" +\n",
    "  f\" --x                            {x_file} \" +\n",
    "  f\" --y_class                      {y_file} \" +\n",
    "  f\" --folding                {folding_file} \" +\n",
    "  f\" --dev                             {dev} \" +\n",
    "  f\" --fold_va                             0 \" +\n",
    "  f\" --fold_te                             1 \" +\n",
    "  f\" --fold_inputs                     32000 \" +\n",
    "  f\" --batch_ratio                      0.01 \" +\n",
    "  f\" --batch_size               {batch_size} \" +\n",
    "  f\" --hidden_sizes                     1000 \" +\n",
    "  f\" --dropouts_trunk                   0.45 \" +\n",
    "  f\" --dropouts_class                      0 \" +\n",
    "  f\" --weight_decay                     1e-4 \" +\n",
    "  f\" --epochs                            100 \" +\n",
    "  f\" --lr                               1e-3 \" +\n",
    "  f\" --lr_steps                           10 \" +\n",
    "  f\" --lr_alpha                          0.3 \" + \n",
    "  f\" --prefix                             sc \" +\n",
    "  f\" --min_samples_class                   5 \"\n",
    ")\n",
    "\n",
    "# f\" --dev              cuda:0 \"\n",
    "# f\" --dev              cuda:0 \""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9201bca",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Initializations "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "784ef5d2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.755160Z",
     "start_time": "2022-11-19T21:27:38.721017Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "  command line parms : \n",
      "------------------------\n",
      " data_dir.................  ../MLDatasets/chembl29_10task\n",
      " output_dir...............  ../experiments/cb29-SparseChem-TaskGroups\n",
      " x........................  chembl_29_X.npy\n",
      " y_class..................  chembl_29_Y_tg_1031_cols_72.npy\n",
      " project_name.............  SC-cb29-TaskGroups\n",
      " exp_id...................  None\n",
      " exp_name.................  None\n",
      " exp_desc.................  \n",
      " folder_sfx...............  None\n",
      " hidden_sizes.............  [1000]\n",
      " dropouts_trunk...........  [0.45]\n",
      " class_feature_size.......  -1\n",
      " last_hidden_sizes........  None\n",
      " epochs...................  100\n",
      " batch_size...............  4096\n",
      " weight_decay.............  0.0001\n",
      " last_non_linearity.......  relu\n",
      " middle_non_linearity.....  relu\n",
      " input_transform..........  none\n",
      " lr.......................  0.001\n",
      " lr_alpha.................  0.3\n",
      " lr_steps.................  [10]\n",
      " weights_class............  None\n",
      " weights_regr.............  None\n",
      " fold_va..................  0\n",
      " fold_te..................  1\n",
      " batch_ratio..............  0.01\n",
      " internal_batch_max.......  None\n",
      " censored_loss............  1\n",
      " folding..................  chembl_29_folding.npy\n",
      " y_regr...................  None\n",
      " y_censor.................  None\n",
      " normalize_loss...........  None\n",
      " normalize_regression.....  0\n",
      " normalize_regr_va........  0\n",
      " inverse_normalization....  0\n",
      " input_size_freq..........  None\n",
      " fold_inputs..............  32000\n",
      " pi_zero..................  0.1\n",
      " min_samples_class........  5\n",
      " min_samples_auc..........  None\n",
      " min_samples_regr.........  10\n",
      " dev......................  cuda:0\n",
      " run_name.................  None\n",
      " prefix...................  sc\n",
      " verbose..................  2\n",
      " save_model...............  1\n",
      " save_board...............  1\n",
      " profile..................  0\n",
      " mixed_precision..........  0\n",
      " eval_train...............  0\n",
      " enable_cat_fusion........  0\n",
      " eval_frequency...........  1\n",
      " regression_weight........  0.5\n",
      " scaling_regularizer......  inf\n",
      " regression_feature_size..  -1\n",
      " last_hidden_sizes_reg....  None\n",
      " last_hidden_sizes_class..  None\n",
      " dropouts_reg.............  []\n",
      " dropouts_class...........  [0.0]\n",
      "\n",
      "\n",
      "\n",
      "../experiments/cb29-SparseChem-TaskGroups/1000x0_1119_1327_lr0.001_do0.45\n",
      "../MLDatasets/chembl29_10task/chembl_29_X.npy\n",
      "../MLDatasets/chembl29_10task/chembl_29_Y_tg_1031_cols_72.npy\n",
      "../MLDatasets/chembl29_10task/chembl_29_folding.npy\n",
      "0 1000\n",
      "Run name is 'sc_1000_lr0.001_do0.45'.\n"
     ]
    }
   ],
   "source": [
    "args = initialize(cmd)\n",
    "def vprint(s=\"\"):\n",
    "    if args.verbose:\n",
    "        print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "01c7b0ed",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.792718Z",
     "start_time": "2022-11-19T21:27:38.760353Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# pp.pprint(vars(args))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4493d6f8",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Assertions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf084ca3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.825112Z",
     "start_time": "2022-11-19T21:27:38.797964Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All assertions passed successfully\n"
     ]
    }
   ],
   "source": [
    "assertions(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3163a92a",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Summary writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7af4ac10",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:38.857066Z",
     "start_time": "2022-11-19T21:27:38.830717Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if args.profile == 1:\n",
    "    assert (args.save_board==1), \"Tensorboard should be enabled to be able to profile memory usage.\"\n",
    "if args.save_board:\n",
    "    # tb_name = os.path.join(args.output_dir, \"\", args.name)\n",
    "    writer  = SummaryWriter(args.output_dir)\n",
    "else:\n",
    "    writer = Nothing()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c1cc295",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Load datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c8eb691c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.093064Z",
     "start_time": "2022-11-19T21:27:38.861499Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row 0 count_nonzero():79\n",
      "\n",
      "task aggresgation weights is None - using min samples rule (= 5)\n",
      " Input dimension   : 32000\n",
      " # Samples         : 423736\n",
      " Y file shape      : (423736, 72)\n",
      "\n",
      "Using 37 classification tasks for calculating aggregated metrics (AUCROC, F1_max, etc).\n",
      "Using 0 regression tasks for calculating metrics (RMSE, Rsquared, correlation).\n"
     ]
    }
   ],
   "source": [
    "ecfp     = sc.load_sparse(args.x)\n",
    "y_class  = sc.load_sparse(args.y_class)\n",
    "y_regr   = sc.load_sparse(args.y_regr)\n",
    "y_censor = sc.load_sparse(args.y_censor)\n",
    "\n",
    "if (y_regr is None) and (y_censor is not None):\n",
    "    raise ValueError(\"y_censor provided please also provide --y_regr.\")\n",
    "if y_class is None:\n",
    "    y_class = scipy.sparse.csr_matrix((ecfp.shape[0], 0))\n",
    "if y_regr is None:\n",
    "    y_regr  = scipy.sparse.csr_matrix((ecfp.shape[0], 0))\n",
    "if y_censor is None:\n",
    "    y_censor = scipy.sparse.csr_matrix(y_regr.shape)\n",
    "\n",
    "# Load folding\n",
    "folding = np.load(args.folding)\n",
    "assert ecfp.shape[0] == folding.shape[0], \"x and folding must have same number of rows\"\n",
    "\n",
    "## Loading task weights\n",
    "tasks_class = sc.load_task_weights(args.weights_class, y=y_class, label=\"y_class\")\n",
    "tasks_regr  = sc.load_task_weights(args.weights_regr, y=y_regr, label=\"y_regr\")\n",
    "\n",
    "#------------------------------------------------------------------\n",
    "## Input and folding transformation\n",
    "#------------------------------------------------------------------\n",
    "ecfp = sc.fold_transform_inputs(ecfp, folding_size=args.fold_inputs, transform=args.input_transform)\n",
    "print(f\"Row 0 count_nonzero():{ecfp[0].count_nonzero()}\")\n",
    "\n",
    "\n",
    "num_pos    = np.array((y_class == +1).sum(0)).flatten()\n",
    "num_neg    = np.array((y_class == -1).sum(0)).flatten()\n",
    "num_class  = np.array((y_class != 0).sum(0)).flatten()\n",
    "if (num_class != num_pos + num_neg).any():\n",
    "    raise ValueError(\"For classification all y values (--y_class/--y) must be 1 or -1.\")\n",
    "\n",
    "num_regr   = np.bincount(y_regr.indices, minlength=y_regr.shape[1])\n",
    "\n",
    "assert args.min_samples_auc is None, \"Parameter 'min_samples_auc' is obsolete. Use '--min_samples_class' that specifies how many samples a task needs per FOLD and per CLASS to be aggregated.\"\n",
    "print()\n",
    "#------------------------------------------------------------------\n",
    "## Aggregation Weights \n",
    "#------------------------------------------------------------------\n",
    "if tasks_class.aggregation_weight is None:\n",
    "    print(f\"task aggresgation weights is None - using min samples rule (= {args.min_samples_class})\")\n",
    "    ## using min_samples rule\n",
    "    fold_pos, fold_neg = sc.class_fold_counts(y_class, folding)\n",
    "    n = args.min_samples_class\n",
    "    tasks_class.aggregation_weight = ((fold_pos >= n).all(0) & (fold_neg >= n)).all(0).astype(np.float64)\n",
    "\n",
    "if tasks_regr.aggregation_weight is None:\n",
    "    if y_censor.nnz == 0:\n",
    "        y_regr2 = y_regr.copy()\n",
    "        y_regr2.data[:] = 1\n",
    "    else:\n",
    "        ## only counting uncensored data\n",
    "        y_regr2      = y_censor.copy()\n",
    "        y_regr2.data = (y_regr2.data == 0).astype(np.int32)\n",
    "    fold_regr, _ = sc.class_fold_counts(y_regr2, folding)\n",
    "    del y_regr2\n",
    "    tasks_regr.aggregation_weight = (fold_regr >= args.min_samples_regr).all(0).astype(np.float64)\n",
    "\n",
    "print(f\" Input dimension   : {ecfp.shape[1]}\")\n",
    "print(f\" # Samples         : {ecfp.shape[0]}\")\n",
    "print(f\" Y file shape      : {y_class.shape}\")\n",
    "print()\n",
    "vprint(f\"Using {(tasks_class.aggregation_weight > 0).sum()} classification tasks for calculating aggregated metrics (AUCROC, F1_max, etc).\")\n",
    "vprint(f\"Using {(tasks_regr.aggregation_weight > 0).sum()} regression tasks for calculating metrics (RMSE, Rsquared, correlation).\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ecb0275c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.128685Z",
     "start_time": "2022-11-19T21:27:39.099762Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold_pos shape is :  (5, 72)\n",
      "fold_pos.sum() shape: (5,),  sum:[3563 4391 3392 3342 3943] \n",
      "\n",
      "fold_neg shape is :  (5, 72)\n",
      "107922 [22246 20633 21917 20736 22390]\n"
     ]
    }
   ],
   "source": [
    "(tasks_class.aggregation_weight > 0).sum()\n",
    "print('fold_pos shape is : ',fold_pos.shape)\n",
    "print(f\"fold_pos.sum() shape: {fold_pos.sum(axis=-1).shape},  sum:{fold_pos.sum(axis=-1)} \")\n",
    "# print(fold_pos)\n",
    "print()\n",
    "print('fold_neg shape is : ',fold_neg.shape)\n",
    "print(fold_neg.sum(), fold_neg.sum(axis=-1))\n",
    "# print(fold_neg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a6da79",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T19:58:14.892601Z",
     "start_time": "2022-04-28T19:58:14.854938Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "97e10187",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.158372Z",
     "start_time": "2022-11-19T21:27:39.132030Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " tasks_class.aggregation_weight.sum:    37.0\n",
      " tasks_class.training_weight.sum   :    72.0\n"
     ]
    }
   ],
   "source": [
    "print(f\" tasks_class.aggregation_weight.sum:    {tasks_class.aggregation_weight.sum()}\")\n",
    "# print(f\" tasks_class.aggregation_weight    :    {tasks_class.aggregation_weight}\")\n",
    "print(f\" tasks_class.training_weight.sum   :    {tasks_class.training_weight.sum()}\")\n",
    "# print(f\" tasks_class.training_weight       :    {tasks_class.training_weight}\")\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c52e01ac",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.330268Z",
     "start_time": "2022-11-19T21:27:39.163256Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "## Separation of test data\n",
    "if args.fold_te is not None and args.fold_te >= 0:\n",
    "    ## removing test data\n",
    "    assert args.fold_te != args.fold_va, \"fold_va and fold_te must not be equal.\"\n",
    "    keep    = folding != args.fold_te\n",
    "    ecfp    = ecfp[keep]\n",
    "    y_class = y_class[keep]\n",
    "    y_regr  = y_regr[keep]\n",
    "    y_censor = y_censor[keep]\n",
    "    folding = folding[keep]\n",
    "\n",
    "## Regression Normalization    \n",
    "normalize_inv = None\n",
    "if args.normalize_regression == 1 and args.normalize_regr_va == 1:\n",
    "   y_regr, mean_save, var_save = sc.normalize_regr(y_regr)\n",
    "\n",
    "## Separation of train and Validation data\n",
    "fold_va = args.fold_va\n",
    "idx_tr  = np.where(folding != fold_va)[0]\n",
    "idx_va  = np.where(folding == fold_va)[0]\n",
    "\n",
    "y_class_tr = y_class[idx_tr]\n",
    "y_class_va = y_class[idx_va]\n",
    "y_regr_tr  = y_regr[idx_tr]\n",
    "y_regr_va  = y_regr[idx_va]\n",
    "y_censor_tr = y_censor[idx_tr]\n",
    "y_censor_va = y_censor[idx_va]\n",
    "\n",
    "## REgression normalization\n",
    "if args.normalize_regression == 1 and args.normalize_regr_va == 0:\n",
    "   y_regr_tr, mean_save, var_save = sc.normalize_regr(y_regr_tr) \n",
    "   if args.inverse_normalization == 1:\n",
    "      normalize_inv = {}\n",
    "      normalize_inv[\"mean\"] = mean_save\n",
    "      normalize_inv[\"var\"]  = var_save\n",
    "    \n",
    "num_pos_va  = np.array((y_class_va == +1).sum(0)).flatten()\n",
    "num_neg_va  = np.array((y_class_va == -1).sum(0)).flatten()\n",
    "num_regr_va = np.bincount(y_regr_va.indices, minlength=y_regr.shape[1])\n",
    "pos_rate = num_pos_va/(num_pos_va+num_neg_va)\n",
    "pos_rate_ref = args.pi_zero\n",
    "pos_rate = np.clip(pos_rate, 0, 0.99)\n",
    "cal_fact_aucpr = pos_rate*(1-pos_rate_ref)/(pos_rate_ref*(1-pos_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f5d2df5b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.462884Z",
     "start_time": "2022-11-19T21:27:39.333543Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " pos_rate  : (72,)    [1.68539326e-01 4.62184874e-02 8.47457627e-03 0.00000000e+00 1.76360225e-01 6.30372493e-02 8.77192982e-03 9.75609756e-04 1.48061105e-01\n",
      " 3.26340326e-02 8.15850816e-03 1.16686114e-03 2.42807018e-01 6.43806858e-02 1.27569100e-02 3.54358611e-03 9.39024390e-01 7.70750988e-01\n",
      " 4.38735178e-01 1.42292490e-01 5.93291405e-01 2.66802444e-01 8.14663951e-02 3.86965377e-02 7.86764706e-01 5.78754579e-01 2.87769784e-01\n",
      " 5.75539568e-02 1.48020654e-01 4.98281787e-02 5.21739130e-03 0.00000000e+00 1.55172414e-01 8.62068966e-03 0.00000000e+00 0.00000000e+00\n",
      " 4.41176471e-02 0.00000000e+00 0.00000000e+00 0.00000000e+00 5.00000000e-01 1.87500000e-01 7.81250000e-02 0.00000000e+00 6.25000000e-01\n",
      " 4.84375000e-01 2.34375000e-01 6.25000000e-02 2.38805970e-01 1.49253731e-02 1.49253731e-02 0.00000000e+00 7.40384615e-01 4.71698113e-01\n",
      " 3.01886792e-01 3.96039604e-02 9.90000000e-01 9.26829268e-01 5.36585366e-01 9.75609756e-02 8.28685259e-01 5.91439689e-01 2.25680934e-01\n",
      " 1.16731518e-02 6.25000000e-01 8.33333333e-02 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00 0.00000000e+00]\n",
      "Input dimension   : 32000\n",
      "Input dimension   : 32000\n",
      "Training dataset  : (254529, 32000)\n",
      "Validation dataset: (82933, 32000)\n",
      "\n",
      "#classification tasks:  72\n",
      "#regression tasks    :      0\n",
      "Using  37 classification tasks for calculating aggregated metrics (AUCROC, F1_max, etc).\n",
      "Using   0 regression tasks for calculating metrics (RMSE, Rsquared, correlation).\n"
     ]
    }
   ],
   "source": [
    "# print(f\" num_pos_va: {num_pos_va.shape}  {num_pos_va}\")\n",
    "# print(f\" num_neg_va: {num_neg_va.shape}  {num_neg_va}\")\n",
    "print(f\" pos_rate  : {pos_rate.shape}    {pos_rate}\")\n",
    "\n",
    "vprint(f\"Input dimension   : {ecfp.shape[1]}\")\n",
    "vprint(f\"Input dimension   : {ecfp.shape[1]}\")\n",
    "vprint(f\"Training dataset  : {ecfp[idx_tr].shape}\")\n",
    "vprint(f\"Validation dataset: {ecfp[idx_va].shape}\")\n",
    "vprint()\n",
    "vprint(f\"#classification tasks:  {y_class.shape[1]}\")\n",
    "vprint(f\"#regression tasks    :      {y_regr.shape[1]}\")\n",
    "vprint(f\"Using {(tasks_class.aggregation_weight > 0).sum():3d} classification tasks for calculating aggregated metrics (AUCROC, F1_max, etc).\")\n",
    "vprint(f\"Using {(tasks_regr.aggregation_weight > 0).sum():3d} regression tasks for calculating metrics (RMSE, Rsquared, correlation).\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "31863130",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Batch Size Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0aefd037",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.505115Z",
     "start_time": "2022-11-19T21:27:39.468568Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "orig batch size:   4096\n",
      "orig num int batches:   1\n",
      "batch size:   4096\n",
      "num_int_batches:   1\n"
     ]
    }
   ],
   "source": [
    "num_int_batches = 1\n",
    "if args.batch_size is not None:\n",
    "    batch_size = args.batch_size\n",
    "else:\n",
    "    batch_size = int(np.ceil(args.batch_ratio * idx_tr.shape[0]))\n",
    "\n",
    "print(f\"orig batch size:   {batch_size}\")\n",
    "print(f\"orig num int batches:   {num_int_batches}\")\n",
    "\n",
    "if args.internal_batch_max is not None:\n",
    "    if args.internal_batch_max < batch_size:\n",
    "        num_int_batches = int(np.ceil(batch_size / args.internal_batch_max))\n",
    "        batch_size      = int(np.ceil(batch_size / num_int_batches))\n",
    "print(f\"batch size:   {batch_size}\")\n",
    "print(f\"num_int_batches:   {num_int_batches}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cec6aa84",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.550876Z",
     "start_time": "2022-11-19T21:27:39.510350Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# #import ipdb; ipdb.set_trace()\n",
    "# batch_size  = int(np.ceil(args.batch_ratio * idx_tr.shape[0]))\n",
    "# num_int_batches = 1\n",
    "\n",
    "# if args.internal_batch_max is not None:\n",
    "#     if args.internal_batch_max < batch_size:\n",
    "#         num_int_batches = int(np.ceil(batch_size / args.internal_batch_max))\n",
    "#         batch_size      = int(np.ceil(batch_size / num_int_batches))\n",
    "# vprint(f\"#internal batch size:   {batch_size}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d4ea279f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.591880Z",
     "start_time": "2022-11-19T21:27:39.557014Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tasks_cat_id_list = None\n",
    "select_cat_ids = None\n",
    "if tasks_class.cat_id is not None:\n",
    "    tasks_cat_id_list = [[x,i] for i,x in enumerate(tasks_class.cat_id) if str(x) != 'nan']\n",
    "    tasks_cat_ids = [i for i,x in enumerate(tasks_class.cat_id) if str(x) != 'nan']\n",
    "    select_cat_ids = np.array(tasks_cat_ids)\n",
    "    cat_id_size = len(tasks_cat_id_list)\n",
    "else:\n",
    "    cat_id_size = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b22caef4",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5d7561ad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.842330Z",
     "start_time": "2022-11-19T21:27:39.596747Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "dataset_tr = sc.ClassRegrSparseDataset(x=ecfp[idx_tr], y_class=y_class_tr, y_regr=y_regr_tr, y_censor=y_censor_tr, y_cat_columns=select_cat_ids)\n",
    "dataset_va = sc.ClassRegrSparseDataset(x=ecfp[idx_va], y_class=y_class_va, y_regr=y_regr_va, y_censor=y_censor_va, y_cat_columns=select_cat_ids)\n",
    "\n",
    "loader_tr = DataLoader(dataset_tr, batch_size=batch_size, num_workers = 8, pin_memory=True, collate_fn=dataset_tr.collate, shuffle=True)\n",
    "loader_va = DataLoader(dataset_va, batch_size=batch_size, num_workers = 4, pin_memory=True, collate_fn=dataset_va.collate, shuffle=False)\n",
    "\n",
    "args.input_size  = dataset_tr.input_size\n",
    "args.output_size = dataset_tr.output_size\n",
    "\n",
    "args.class_output_size = dataset_tr.class_output_size\n",
    "args.regr_output_size  = dataset_tr.regr_output_size\n",
    "args.cat_id_size = cat_id_size\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3bd2ad04",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:39.871723Z",
     "start_time": "2022-11-19T21:27:39.844783Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " dataset_tr.y_class                                 :  (254529, 72) \n",
      " dataset_va.y_class                                 :  (82933, 72) \n",
      "                                 \n",
      " size of training set                               :  254529 \n",
      " size of validation set                             :  82933 \n",
      "                                 \n",
      " Number of batches in training                      :  63 \n",
      " Number of batches in validation dataset            :  21 \n",
      "                                \n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(f\"\\n dataset_tr.y_class                                 :  {dataset_tr.y_class.shape}\",\n",
    "      f\"\\n dataset_va.y_class                                 :  {dataset_va.y_class.shape}\",\n",
    "#       f\"\\n dataset_test.y_class                                 :  {dataset_va.y_class.shape}\",\n",
    "      f\"\\n                                \",\n",
    "      f'\\n size of training set                               :  {len(dataset_tr)}',\n",
    "      f'\\n size of validation set                             :  {len(dataset_va)}',\n",
    "#     #   f'\\n size of test set                                   :  {len(dldrs.testset)}',\n",
    "#     #   f'\\n                               Total                :  {len(dldrs.trainset0)+len(dldrs.trainset1)+len(dldrs.trainset2)+len(dldrs.valset)+ len(dldrs.testset)}',\n",
    "      f\"\\n                                \",\n",
    "      f\"\\n Number of batches in training                      :  {len(loader_tr)}\",\n",
    "      f\"\\n Number of batches in validation dataset            :  {len(loader_va)}\",\n",
    "    #   f\"\\n lenght (# batches) in test dataset                 :  {len(dldrs.test_loader)}\",\n",
    "      f\"\\n                                \")\n",
    "                "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6868699b",
   "metadata": {
    "hidden": true
   },
   "source": [
    "###  WandB setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0f275b83",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:47.021362Z",
     "start_time": "2022-11-19T21:27:39.875713Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2i0valte 1119_1327 SC-cb29-TaskGroups\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mkbardool\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.13.5 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.13.4"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/home/kbardool/WSL-projs/SparseChem/wandb/run-20221119_132741-2i0valte</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/kbardool/SC-cb29-TaskGroups/runs/2i0valte\" target=\"_blank\">1119_1327</a></strong> to <a href=\"https://wandb.ai/kbardool/SC-cb29-TaskGroups\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " PROJECT NAME: SC-cb29-TaskGroups\n",
      " RUN ID      : 2i0valte \n",
      " RUN NAME    : 1119_1327\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<wandb.sdk.wandb_metric.Metric at 0x7f653f404a00>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#------------------------------------------------------------------\n",
    "# ### WandB setup\n",
    "#------------------------------------------------------------------\n",
    "ns = types.SimpleNamespace()\n",
    "ns.current_epoch  = 0\n",
    "ns.current_iter   = 0\n",
    "ns.best_results   = {}\n",
    "ns.best_metrics   = None\n",
    "ns.best_accuracy  = 0 \n",
    "ns.best_roc_auc   = 0 \n",
    "ns.best_iter      = 0\n",
    "ns.best_epoch     = 0\n",
    "ns.p_epoch        = 0\n",
    "ns.num_prints     = 0\n",
    "\n",
    "init_wandb(ns, args)\n",
    "wandb.define_metric(\"best_accuracy\", summary=\"last\")\n",
    "wandb.define_metric(\"best_roc_auc\", summary=\"last\")\n",
    "wandb.define_metric(\"best_epoch\", summary=\"last\")\n",
    "\n",
    "# ns.best_value     = 0 \n",
    "# wandb.define_metric(\"best_accuracy\", summary=\"last\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b834837",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "00ec1303",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:49.970200Z",
     "start_time": "2022-11-19T21:27:47.026051Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kbardool/miniconda3/envs/pyt-gpu/lib/python3.9/site-packages/torch/nn/init.py:403: UserWarning: Initializing zero-element tensors is a no-op\n",
      "  warnings.warn(\"Initializing zero-element tensors is a no-op\")\n"
     ]
    }
   ],
   "source": [
    "#------------------------------------------------------------------\n",
    "# ### Network\n",
    "#------------------------------------------------------------------\n",
    "dev  = torch.device(args.dev)\n",
    "\n",
    "net  = sc.SparseFFN(args).to(dev)\n",
    "loss_class = torch.nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "loss_regr  = sc.censored_mse_loss\n",
    "\n",
    "if not args.censored_loss:\n",
    "    loss_regr = functools.partial(loss_regr, censored_enabled=False)\n",
    "\n",
    "tasks_class.training_weight = tasks_class.training_weight.to(dev)\n",
    "tasks_regr.training_weight  = tasks_regr.training_weight.to(dev)\n",
    "tasks_regr.censored_weight  = tasks_regr.censored_weight.to(dev)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c0b4f16",
   "metadata": {
    "hidden": true
   },
   "source": [
    "###  Optimizer, Scheduler, GradScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ebcc9050",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:50.003500Z",
     "start_time": "2022-11-19T21:27:49.973506Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#------------------------------------------------------------------\n",
    "# ###  Optimizer, Scheduler, GradScaler\n",
    "#------------------------------------------------------------------\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=args.lr, weight_decay=args.weight_decay)\n",
    "scheduler = MultiStepLR(optimizer, milestones=args.lr_steps, gamma=args.lr_alpha)\n",
    "scaler = torch.cuda.amp.GradScaler()\n",
    "\n",
    "wandb.watch(net, log='all', log_freq= 100)     ###  Weights and Biases Initialization \n",
    "reporter = None\n",
    "h = None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "080e975d",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### setup memory profiling reporter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5f357697",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:50.034954Z",
     "start_time": "2022-11-19T21:27:50.008192Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "if args.profile == 1:\n",
    "   torch_gpu_id = torch.cuda.current_device()\n",
    "   if \"CUDA_VISIBLE_DEVICES\" in os.environ:\n",
    "      ids = list(map(int, os.environ.get(\"CUDA_VISIBLE_DEVICES\", \"\").split(\",\")))\n",
    "      nvml_gpu_id = ids[torch_gpu_id] # remap\n",
    "   else:\n",
    "      nvml_gpu_id = torch_gpu_id\n",
    "   h = nvmlDeviceGetHandleByIndex(nvml_gpu_id)\n",
    "\n",
    "if args.profile == 1:\n",
    "   #####   output saving   #####\n",
    "   if not os.path.exists(args.output_dir):\n",
    "       os.makedirs(args.output_dir)\n",
    "\n",
    "   reporter = MemReporter(net)\n",
    "\n",
    "   with open(f\"{args.output_dir}/memprofile.txt\", \"w+\") as profile_file:\n",
    "        with redirect_stdout(profile_file):\n",
    "             profile_file.write(f\"\\nInitial model detailed report:\\n\\n\")\n",
    "             reporter.report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "810b495f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:50.072679Z",
     "start_time": "2022-11-19T21:27:50.038651Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network:\n",
      "SparseFFN(\n",
      "  (net): Sequential(\n",
      "    (0): SparseInputNet(\n",
      "      (net_freq): SparseLinear(in_features=32000, out_features=1000, bias=True)\n",
      "    )\n",
      "    (1): MiddleNet(\n",
      "      (net): Sequential()\n",
      "    )\n",
      "  )\n",
      "  (classLast): LastNet(\n",
      "    (net): Sequential(\n",
      "      (initial_layer): Sequential(\n",
      "        (0): ReLU()\n",
      "        (1): Dropout(p=0.45, inplace=False)\n",
      "        (2): Linear(in_features=1000, out_features=72, bias=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (regrLast): Sequential(\n",
      "    (0): LastNet(\n",
      "      (net): Sequential(\n",
      "        (initial_layer): Sequential(\n",
      "          (0): Tanh()\n",
      "          (1): Dropout(p=0.45, inplace=False)\n",
      "          (2): Linear(in_features=1000, out_features=0, bias=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "  )\n",
      ")\n",
      "Adam (\n",
      "Parameter Group 0\n",
      "    amsgrad: False\n",
      "    betas: (0.9, 0.999)\n",
      "    eps: 1e-08\n",
      "    initial_lr: 0.001\n",
      "    lr: 0.001\n",
      "    maximize: False\n",
      "    weight_decay: 0.0001\n",
      ")\n",
      "dev                  :    cuda:0\n",
      "args.lr              :    0.001\n",
      "args.weight_decay    :    0.0001\n",
      "args.lr_steps        :    [10]\n",
      "args.lr_steps        :    [10]\n",
      "num_int_batches      :    1\n",
      "batch_size           :    4096\n",
      "current epoch        :    0\n",
      "epochs               :    100\n",
      "scaler               :    <torch.cuda.amp.grad_scaler.GradScaler object at 0x7f65412952b0>\n",
      "args.normalize_loss  :    None\n",
      "loss_class           :    BCEWithLogitsLoss()\n",
      "mixed precision      :    0\n",
      "args.eval_train      :    0\n"
     ]
    }
   ],
   "source": [
    "#------------------------------------------------------------------\n",
    "# ### Display network and other values\n",
    "#------------------------------------------------------------------\n",
    "print(\"Network:\")\n",
    "print(net)\n",
    "print(optimizer)\n",
    "print(f\"dev                  :    {dev}\")\n",
    "print(f\"args.lr              :    {args.lr}\")\n",
    "print(f\"args.weight_decay    :    {args.weight_decay}\")\n",
    "print(f\"args.lr_steps        :    {args.lr_steps}\")\n",
    "print(f\"args.lr_steps        :    {args.lr_steps}\")\n",
    "print(f\"num_int_batches      :    {num_int_batches}\")\n",
    "print(f\"batch_size           :    {batch_size}\")\n",
    "print(f\"current epoch        :    {ns.current_epoch}\")\n",
    "print(f\"epochs               :    {args.epochs}\")\n",
    "print(f\"scaler               :    {scaler}\")\n",
    "print(f\"args.normalize_loss  :    {args.normalize_loss}\")\n",
    "print(f\"loss_class           :    {loss_class}\")\n",
    "print(f\"mixed precision      :    {args.mixed_precision}\")\n",
    "print(f\"args.eval_train      :    {args.eval_train}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64267731",
   "metadata": {},
   "source": [
    "##  Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "628e976d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:27:50.105275Z",
     "start_time": "2022-11-19T21:27:50.077025Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Last Epoch: 0   # of epochs to do:  100 - Run epochs 1 to 100\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "# from torch.serialization import SourceChangeWarning \n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)    \n",
    "ns.end_epoch = ns.current_epoch + args.epochs\n",
    "print(f\" Last Epoch: {ns.current_epoch}   # of epochs to do:  {args.epochs} - Run epochs {ns.current_epoch+1} to {ns.end_epoch}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9446ed45",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b9f8fa8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:52:18.535949Z",
     "start_time": "2022-11-19T21:27:50.108919Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  |      logl   bceloss  avg prec   auc roc    auc pr aucpr_cal    f1_max |      rmse  rsquared  corrcoef | tr_time \n",
      "1      |   0.22395   0.38861   0.57439   0.75645   0.56915   0.41418   0.59657 |       nan       nan       nan |   12.9 \n",
      " Previous best_epoch:     0       best_accuracy: 0.00000    best ROC auc: 0.00000\n",
      " New      best_epoch:     1       best_accuracy: 0.57439    best ROC auc: 0.75645\n",
      "2      |   0.21454   0.36373   0.58454   0.78039   0.57804   0.43509   0.60648 |       nan       nan       nan |   13.4 \n",
      " Previous best_epoch:     1       best_accuracy: 0.57439    best ROC auc: 0.75645\n",
      " New      best_epoch:     2       best_accuracy: 0.58454    best ROC auc: 0.78039\n",
      "3      |   0.21905   0.37615   0.58113   0.77918   0.57395   0.42978   0.60798 |       nan       nan       nan |   12.6 \n",
      "4      |   0.21760   0.36173   0.58357   0.78543   0.57597   0.43158   0.61026 |       nan       nan       nan |   14.0 \n",
      " Previous best_epoch:     2       best_accuracy: 0.58454    best ROC auc: 0.78039\n",
      " New      best_epoch:     4       best_accuracy: 0.58357    best ROC auc: 0.78543\n",
      "5      |   0.21233   0.35783   0.59616   0.79472   0.59000   0.44692   0.62169 |       nan       nan       nan |   12.8 \n",
      " Previous best_epoch:     4       best_accuracy: 0.58357    best ROC auc: 0.78543\n",
      " New      best_epoch:     5       best_accuracy: 0.59616    best ROC auc: 0.79472\n",
      "6      |   0.21334   0.36097   0.58521   0.79269   0.57691   0.43988   0.61970 |       nan       nan       nan |   12.1 \n",
      "7      |   0.21410   0.36032   0.58797   0.79474   0.58126   0.43513   0.61564 |       nan       nan       nan |   12.8 \n",
      " Previous best_epoch:     5       best_accuracy: 0.59616    best ROC auc: 0.79472\n",
      " New      best_epoch:     7       best_accuracy: 0.58797    best ROC auc: 0.79474\n",
      "8      |   0.21444   0.36156   0.58381   0.79345   0.57579   0.43297   0.61422 |       nan       nan       nan |   13.4 \n",
      "9      |   0.21601   0.36151   0.58610   0.79639   0.57984   0.43727   0.61665 |       nan       nan       nan |   12.6 \n",
      " Previous best_epoch:     7       best_accuracy: 0.58797    best ROC auc: 0.79474\n",
      " New      best_epoch:     9       best_accuracy: 0.58610    best ROC auc: 0.79639\n",
      "10     |   0.21624   0.36433   0.58368   0.79758   0.57647   0.43454   0.61341 |       nan       nan       nan |   13.4 \n",
      " Previous best_epoch:     9       best_accuracy: 0.58610    best ROC auc: 0.79639\n",
      " New      best_epoch:    10       best_accuracy: 0.58368    best ROC auc: 0.79758\n",
      "11     |   0.21283   0.35978   0.59056   0.79912   0.58327   0.44154   0.61588 |       nan       nan       nan |   12.9 \n",
      " Previous best_epoch:    10       best_accuracy: 0.58368    best ROC auc: 0.79758\n",
      " New      best_epoch:    11       best_accuracy: 0.59056    best ROC auc: 0.79912\n",
      "12     |   0.21269   0.35643   0.58898   0.79975   0.58045   0.44044   0.61522 |       nan       nan       nan |   12.9 \n",
      " Previous best_epoch:    11       best_accuracy: 0.59056    best ROC auc: 0.79912\n",
      " New      best_epoch:    12       best_accuracy: 0.58898    best ROC auc: 0.79975\n",
      "13     |   0.21179   0.35702   0.59105   0.80077   0.58262   0.44343   0.61684 |       nan       nan       nan |   12.9 \n",
      " Previous best_epoch:    12       best_accuracy: 0.58898    best ROC auc: 0.79975\n",
      " New      best_epoch:    13       best_accuracy: 0.59105    best ROC auc: 0.80077\n",
      "14     |   0.21258   0.35559   0.59049   0.80166   0.58214   0.44409   0.61522 |       nan       nan       nan |   12.6 \n",
      " Previous best_epoch:    13       best_accuracy: 0.59105    best ROC auc: 0.80077\n",
      " New      best_epoch:    14       best_accuracy: 0.59049    best ROC auc: 0.80166\n",
      "15     |   0.21188   0.35463   0.59596   0.80463   0.58891   0.44781   0.61822 |       nan       nan       nan |   13.4 \n",
      " Previous best_epoch:    14       best_accuracy: 0.59049    best ROC auc: 0.80166\n",
      " New      best_epoch:    15       best_accuracy: 0.59596    best ROC auc: 0.80463\n",
      "16     |   0.21303   0.35612   0.59243   0.80044   0.58555   0.44462   0.61569 |       nan       nan       nan |   13.5 \n",
      "17     |   0.21093   0.35296   0.59423   0.80320   0.58709   0.44598   0.61802 |       nan       nan       nan |   12.7 \n",
      "18     |   0.21123   0.35454   0.59798   0.80418   0.59101   0.45026   0.62216 |       nan       nan       nan |   12.9 \n",
      "19     |   0.21293   0.35409   0.59172   0.80120   0.58347   0.44274   0.61910 |       nan       nan       nan |   12.1 \n",
      "20     |   0.21446   0.35952   0.59176   0.80000   0.58367   0.44453   0.61739 |       nan       nan       nan |   13.3 \n",
      "Epoch  |      logl   bceloss  avg prec   auc roc    auc pr aucpr_cal    f1_max |      rmse  rsquared  corrcoef | tr_time \n",
      "21     |   0.21418   0.35888   0.59324   0.80112   0.58624   0.44664   0.61753 |       nan       nan       nan |   13.4 \n",
      "22     |   0.21456   0.35912   0.59050   0.80146   0.58197   0.44279   0.61851 |       nan       nan       nan |   12.8 \n",
      "23     |   0.21402   0.35946   0.59376   0.80242   0.58619   0.44558   0.61701 |       nan       nan       nan |   13.7 \n",
      "24     |   0.21586   0.36155   0.59002   0.80099   0.58281   0.44185   0.61542 |       nan       nan       nan |   13.1 \n",
      "25     |   0.21484   0.35988   0.59340   0.80321   0.58686   0.44579   0.61765 |       nan       nan       nan |   12.1 \n",
      "26     |   0.21502   0.35969   0.59598   0.80316   0.58870   0.44916   0.61620 |       nan       nan       nan |   12.8 \n",
      "27     |   0.21524   0.36142   0.59449   0.80280   0.58636   0.44725   0.61889 |       nan       nan       nan |   13.4 \n",
      "28     |   0.21512   0.35941   0.59420   0.80384   0.58718   0.44951   0.61356 |       nan       nan       nan |   12.6 \n",
      "29     |   0.21615   0.36177   0.59087   0.80058   0.58277   0.44296   0.61669 |       nan       nan       nan |   13.6 \n",
      "30     |   0.21591   0.35978   0.58493   0.80147   0.57604   0.43577   0.61624 |       nan       nan       nan |   12.1 \n",
      "31     |   0.21422   0.35926   0.59324   0.80260   0.58620   0.44449   0.61716 |       nan       nan       nan |   12.8 \n",
      "32     |   0.21340   0.35831   0.59974   0.80242   0.59294   0.45318   0.62398 |       nan       nan       nan |   12.8 \n",
      "33     |   0.21640   0.36078   0.58878   0.80165   0.58040   0.44197   0.61606 |       nan       nan       nan |   12.8 \n",
      "34     |   0.21716   0.36306   0.59535   0.80111   0.58866   0.44726   0.61977 |       nan       nan       nan |   13.6 \n",
      "35     |   0.21631   0.36187   0.59081   0.80146   0.58286   0.44267   0.61490 |       nan       nan       nan |   13.3 \n",
      "36     |   0.21720   0.36271   0.58873   0.80174   0.58072   0.44075   0.61515 |       nan       nan       nan |   12.8 \n",
      "37     |   0.21638   0.36279   0.59180   0.79988   0.58497   0.44306   0.61502 |       nan       nan       nan |   13.0 \n",
      "38     |   0.21608   0.36113   0.59151   0.80104   0.58348   0.44198   0.61553 |       nan       nan       nan |   12.1 \n",
      "39     |   0.21663   0.35942   0.59303   0.80263   0.58609   0.44296   0.61338 |       nan       nan       nan |   13.5 \n",
      "40     |   0.21648   0.36195   0.59410   0.80256   0.58730   0.44573   0.61836 |       nan       nan       nan |   13.3 \n",
      "Epoch  |      logl   bceloss  avg prec   auc roc    auc pr aucpr_cal    f1_max |      rmse  rsquared  corrcoef | tr_time \n",
      "41     |   0.21625   0.35918   0.59694   0.80343   0.59018   0.44788   0.62054 |       nan       nan       nan |   12.6 \n",
      "42     |   0.21730   0.36386   0.59274   0.80313   0.58558   0.44435   0.61464 |       nan       nan       nan |   13.5 \n",
      "43     |   0.21693   0.36084   0.59686   0.80469   0.58998   0.44937   0.61820 |       nan       nan       nan |   12.9 \n",
      " Previous best_epoch:    15       best_accuracy: 0.59596    best ROC auc: 0.80463\n",
      " New      best_epoch:    43       best_accuracy: 0.59686    best ROC auc: 0.80469\n",
      "44     |   0.21789   0.36198   0.59188   0.80249   0.58463   0.44267   0.61654 |       nan       nan       nan |   12.2 \n",
      "45     |   0.21750   0.36274   0.59557   0.80382   0.58880   0.44695   0.61512 |       nan       nan       nan |   13.4 \n",
      "46     |   0.21845   0.36149   0.59160   0.80262   0.58488   0.44374   0.61557 |       nan       nan       nan |   12.8 \n",
      "47     |   0.21769   0.36424   0.59466   0.80274   0.58770   0.44622   0.61702 |       nan       nan       nan |   13.4 \n",
      "48     |   0.21708   0.36209   0.59176   0.80444   0.58367   0.44306   0.61732 |       nan       nan       nan |   13.5 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49     |   0.21686   0.36049   0.59409   0.80473   0.58682   0.44581   0.61730 |       nan       nan       nan |   12.1 \n",
      " Previous best_epoch:    43       best_accuracy: 0.59686    best ROC auc: 0.80469\n",
      " New      best_epoch:    49       best_accuracy: 0.59409    best ROC auc: 0.80473\n",
      "50     |   0.21856   0.36353   0.58831   0.80291   0.58042   0.43963   0.61497 |       nan       nan       nan |   12.9 \n",
      "51     |   0.21592   0.36222   0.59784   0.80395   0.59123   0.44846   0.62033 |       nan       nan       nan |   12.9 \n",
      "52     |   0.21852   0.36314   0.59548   0.80343   0.58878   0.44592   0.61623 |       nan       nan       nan |   12.5 \n",
      "53     |   0.21728   0.36506   0.59426   0.80422   0.58626   0.44464   0.61965 |       nan       nan       nan |   13.6 \n",
      "54     |   0.21938   0.36475   0.59368   0.80255   0.58683   0.44557   0.61854 |       nan       nan       nan |   13.6 \n",
      "55     |   0.21853   0.36402   0.59315   0.80233   0.58588   0.44378   0.61815 |       nan       nan       nan |   12.0 \n",
      "56     |   0.21710   0.36302   0.59768   0.80309   0.59123   0.45030   0.61865 |       nan       nan       nan |   12.8 \n",
      "57     |   0.22105   0.36765   0.59236   0.80038   0.58447   0.44664   0.61511 |       nan       nan       nan |   12.2 \n",
      "58     |   0.21942   0.36498   0.59284   0.80363   0.58610   0.44564   0.61583 |       nan       nan       nan |   13.6 \n",
      "59     |   0.21929   0.36321   0.59504   0.80079   0.58835   0.44912   0.61539 |       nan       nan       nan |   13.5 \n",
      "60     |   0.21924   0.36501   0.59494   0.80165   0.58868   0.44758   0.61510 |       nan       nan       nan |   12.8 \n",
      "Epoch  |      logl   bceloss  avg prec   auc roc    auc pr aucpr_cal    f1_max |      rmse  rsquared  corrcoef | tr_time \n",
      "61     |   0.21852   0.36468   0.59257   0.80144   0.58562   0.44421   0.61627 |       nan       nan       nan |   13.7 \n",
      "62     |   0.21853   0.36357   0.59695   0.80168   0.59012   0.45051   0.61988 |       nan       nan       nan |   12.9 \n",
      "63     |   0.21914   0.36293   0.59458   0.80130   0.58762   0.44628   0.61793 |       nan       nan       nan |   12.2 \n",
      "64     |   0.21947   0.36674   0.59124   0.80139   0.58448   0.44308   0.61373 |       nan       nan       nan |   13.4 \n",
      "65     |   0.21899   0.36539   0.59676   0.80174   0.58983   0.45069   0.61924 |       nan       nan       nan |   12.7 \n",
      "66     |   0.21968   0.36766   0.59178   0.80120   0.58480   0.44112   0.61579 |       nan       nan       nan |   13.6 \n",
      "67     |   0.21946   0.36321   0.59417   0.80128   0.58739   0.44712   0.61671 |       nan       nan       nan |   13.6 \n",
      "68     |   0.22124   0.36666   0.58985   0.80094   0.58292   0.43993   0.61439 |       nan       nan       nan |   12.0 \n",
      "69     |   0.21866   0.36444   0.58925   0.80228   0.58165   0.44138   0.61552 |       nan       nan       nan |   13.0 \n",
      "70     |   0.22123   0.36673   0.58738   0.80034   0.58033   0.43671   0.61494 |       nan       nan       nan |   13.5 \n",
      "71     |   0.22162   0.36832   0.58797   0.80002   0.58078   0.43882   0.61210 |       nan       nan       nan |   13.1 \n",
      "72     |   0.22025   0.36623   0.59370   0.80150   0.58682   0.44570   0.61689 |       nan       nan       nan |   13.5 \n",
      "73     |   0.21969   0.36625   0.58999   0.80033   0.58264   0.44127   0.61437 |       nan       nan       nan |   12.7 \n",
      "74     |   0.22214   0.36925   0.58916   0.80094   0.58216   0.44133   0.61743 |       nan       nan       nan |   12.9 \n",
      "75     |   0.22234   0.36711   0.58762   0.79879   0.58069   0.43722   0.61425 |       nan       nan       nan |   12.9 \n",
      "76     |   0.22088   0.36827   0.58926   0.79881   0.58214   0.44004   0.61517 |       nan       nan       nan |   12.1 \n",
      "77     |   0.22146   0.36864   0.58888   0.80038   0.58183   0.44088   0.61295 |       nan       nan       nan |   13.5 \n",
      "78     |   0.21862   0.36298   0.59453   0.80334   0.58828   0.44671   0.61598 |       nan       nan       nan |   13.6 \n",
      "79     |   0.21990   0.36395   0.59080   0.80183   0.58397   0.44498   0.61355 |       nan       nan       nan |   12.6 \n",
      "80     |   0.22178   0.36612   0.59274   0.80083   0.58601   0.44508   0.61578 |       nan       nan       nan |   12.9 \n",
      "Epoch  |      logl   bceloss  avg prec   auc roc    auc pr aucpr_cal    f1_max |      rmse  rsquared  corrcoef | tr_time \n",
      "81     |   0.21895   0.36269   0.59180   0.80195   0.58463   0.44440   0.61637 |       nan       nan       nan |   13.0 \n",
      "82     |   0.22063   0.36582   0.58943   0.80095   0.58235   0.44066   0.61340 |       nan       nan       nan |   12.1 \n",
      "83     |   0.21868   0.36214   0.59872   0.80120   0.59248   0.45213   0.62121 |       nan       nan       nan |   13.5 \n",
      "84     |   0.22219   0.36687   0.58598   0.79914   0.57909   0.43866   0.61258 |       nan       nan       nan |   12.7 \n",
      "85     |   0.21933   0.36354   0.58253   0.80045   0.57418   0.43481   0.61211 |       nan       nan       nan |   13.5 \n",
      "86     |   0.22075   0.36721   0.59388   0.80083   0.58744   0.44594   0.61490 |       nan       nan       nan |   13.6 \n",
      "87     |   0.22104   0.36863   0.58830   0.79997   0.58070   0.43983   0.61178 |       nan       nan       nan |   12.0 \n",
      "88     |   0.22163   0.36644   0.58845   0.79894   0.58074   0.43966   0.61323 |       nan       nan       nan |   13.0 \n",
      "89     |   0.21924   0.36569   0.58741   0.80082   0.57942   0.43798   0.61529 |       nan       nan       nan |   13.6 \n",
      "90     |   0.22113   0.36567   0.59340   0.80054   0.58665   0.44774   0.61506 |       nan       nan       nan |   12.8 \n",
      "91     |   0.22146   0.36734   0.58700   0.80037   0.57873   0.43742   0.61289 |       nan       nan       nan |   13.6 \n",
      "92     |   0.22162   0.36502   0.58835   0.80140   0.58037   0.44064   0.61480 |       nan       nan       nan |   12.8 \n",
      "93     |   0.22060   0.36581   0.58872   0.80126   0.58046   0.44074   0.61335 |       nan       nan       nan |   12.8 \n",
      "94     |   0.22014   0.36352   0.59213   0.80314   0.58487   0.44340   0.61531 |       nan       nan       nan |   12.8 \n",
      "95     |   0.22014   0.36616   0.59270   0.80111   0.58602   0.44546   0.61533 |       nan       nan       nan |   13.1 \n",
      "96     |   0.22088   0.36602   0.59148   0.80191   0.58469   0.44310   0.61538 |       nan       nan       nan |   13.7 \n",
      "97     |   0.22089   0.36482   0.59529   0.80239   0.58851   0.44652   0.61314 |       nan       nan       nan |   13.5 \n",
      "98     |   0.22290   0.37092   0.58967   0.79875   0.58273   0.44001   0.61487 |       nan       nan       nan |   12.8 \n",
      "99     |   0.22321   0.36857   0.59052   0.79925   0.58354   0.44053   0.61316 |       nan       nan       nan |   12.9 \n",
      "last_round\n",
      " roc_auc_score         0.7993\n",
      " auc_pr                0.5835\n",
      " avg_prec_score        0.5905\n",
      " f1_max                0.6132\n",
      " p_f1_max              0.2992\n",
      " kappa                 0.2595\n",
      " kappa_max             0.4406\n",
      " p_kappa_max           0.3435\n",
      " bceloss               0.3686\n",
      " auc_pr_cal            0.4405\n",
      " logloss               0.2232\n",
      " num_tasks_total       72.0000\n",
      " num_tasks_agg         37.0000\n",
      "100    |   0.22140   0.36495   0.59387   0.80168   0.58707   0.44584   0.61487 |       nan       nan       nan |   12.9 \n",
      "Best Epoch :       49\n",
      "Best Iteration :   0 \n",
      "Best Precision :   0.59409\n",
      "Best ROC AUC   :   0.80473\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ns.end_epoch = ns.current_epoch + args.epochs\n",
    "\n",
    "for ns.current_epoch in range(ns.current_epoch+1, ns.end_epoch+1, 1):\n",
    "    t0 = time.time()\n",
    "    sc.train_class_regr(\n",
    "        net, optimizer,\n",
    "        loader          = loader_tr,\n",
    "        loss_class      = loss_class,\n",
    "        loss_regr       = loss_regr,\n",
    "        dev             = dev,\n",
    "        weights_class   = tasks_class.training_weight * (1-args.regression_weight) * 2,\n",
    "        weights_regr    = tasks_regr.training_weight * args.regression_weight * 2,\n",
    "        censored_weight = tasks_regr.censored_weight,\n",
    "        normalize_loss  = args.normalize_loss,\n",
    "        num_int_batches = num_int_batches,\n",
    "        progress        = False,\n",
    "        writer          = writer,\n",
    "        epoch           = ns.current_epoch,\n",
    "        args            = args,\n",
    "        scaler          = scaler,\n",
    "        nvml_handle     = h)\n",
    "\n",
    "    if args.profile == 1:\n",
    "       with open(f\"{args.output_dir}/memprofile.txt\", \"a+\") as profile_file:\n",
    "            profile_file.write(f\"\\nAfter epoch {epoch} model detailed report:\\n\\n\")\n",
    "            with redirect_stdout(profile_file):\n",
    "                 reporter.report()\n",
    "\n",
    "    t1 = time.time()\n",
    "    eval_round = (args.eval_frequency > 0) and ((ns.current_epoch + 1) % args.eval_frequency == 0)\n",
    "    last_round = ns.current_epoch == args.epochs - 1\n",
    "\n",
    "    if eval_round or last_round:\n",
    "\n",
    "        results_va = sc.evaluate_class_regr(net, loader_va, loss_class, loss_regr, \n",
    "                                            tasks_class= tasks_class, \n",
    "                                            tasks_regr = tasks_regr, \n",
    "                                            dev        = dev, \n",
    "                                            progress   = False, \n",
    "                                            normalize_inv=normalize_inv, \n",
    "                                            cal_fact_aucpr=cal_fact_aucpr)\n",
    "        \n",
    "        for key, val in results_va[\"classification_agg\"].items():\n",
    "            writer.add_scalar(\"val_metrics:aggregated/\"+key, val, ns.current_epoch * batch_size)\n",
    "\n",
    "\n",
    "        if args.eval_train:\n",
    "            results_tr = sc.evaluate_class_regr(net, loader_tr, loss_class, loss_regr, \n",
    "                                                tasks_class = tasks_class, \n",
    "                                                tasks_regr  = tasks_regr, \n",
    "                                                dev         = dev, \n",
    "                                                progress    = args.verbose >= 2)\n",
    "            for key, val in results_tr[\"classification_agg\"].items():\n",
    "                writer.add_scalar(\"trn_metrics:aggregated/\"+key, val, ns.current_epoch * batch_size)\n",
    "\n",
    "        else:\n",
    "            results_tr = None\n",
    "\n",
    "        if args.verbose:\n",
    "            ## printing a new header every 20 lines\n",
    "            header = ns.num_prints % 20 == 0\n",
    "            ns.num_prints += 1\n",
    "            sc.print_metrics_cr(ns.current_epoch, t1 - t0, results_tr, results_va, header)\n",
    "            \n",
    "        wandb.log(results_va[\"classification_agg\"].to_dict())\n",
    "\n",
    "        check_for_improvement(ns, results_va)\n",
    "\n",
    "    scheduler.step()\n",
    "    \n",
    "    if last_round:\n",
    "        print('last_round')\n",
    "        for k in results_va['classification_agg'].keys():\n",
    "            print(f\" {k:20s}  {results_va['classification_agg'][k]:.4f}\")\n",
    "    \n",
    "    \n",
    "    \n",
    "print(f\"Best Epoch :       {ns.best_epoch}\\n\"\n",
    "      f\"Best Iteration :   {ns.best_iter} \\n\"\n",
    "      f\"Best Precision :   {ns.best_accuracy:.5f}\\n\"\n",
    "      f\"Best ROC AUC   :   {ns.best_roc_auc:.5f}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2958c7a9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85747536",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9e5a36fa",
   "metadata": {},
   "source": [
    "## Post Training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f24a0d9d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:52:18.800532Z",
     "start_time": "2022-11-19T21:52:18.541127Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Saving performance metrics (AUCs) and model.\n",
      "Saved model weights into '../experiments/cb29-SparseChem-TaskGroups/1000x0_1119_1327_lr0.001_do0.45/sc_1000_lr0.001_do0.45.pt'.\n",
      "Saved config and results into '../experiments/cb29-SparseChem-TaskGroups/1000x0_1119_1327_lr0.001_do0.45/sc_1000_lr0.001_do0.45.json'.\n",
      "You can load the results by:\n",
      "  import sparsechem as sc\n",
      "  res = sc.load_results('../experiments/cb29-SparseChem-TaskGroups/1000x0_1119_1327_lr0.001_do0.45/sc_1000_lr0.001_do0.45.json')\n"
     ]
    }
   ],
   "source": [
    "#print(\"DEBUG data for hidden spliting\")\n",
    "#print (f\"Classification mask: Sum = {net.classmask.sum()}\\t Uniques: {np.unique(net.classmask)}\")\n",
    "#print (f\"Regression mask:     Sum = {net.regmask.sum()}\\t Uniques: {np.unique(net.regmask)}\")\n",
    "#print (f\"overlap: {(net.regmask * net.classmask).sum()}\")\n",
    "\n",
    "writer.close()\n",
    "vprint()\n",
    "if args.profile == 1:\n",
    "   multiplexer = sc.create_multiplexer(tb_name)\n",
    "#   sc.export_scalars(multiplexer, '.', \"GPUmem\", \"testcsv.csv\")\n",
    "   data = sc.extract_scalars(multiplexer, '.', \"GPUmem\")\n",
    "   vprint(f\"Peak GPU memory used: {sc.return_max_val(data)}MB\")\n",
    "vprint(\"Saving performance metrics (AUCs) and model.\")\n",
    "\n",
    "#####   model saving   #####\n",
    "if not os.path.exists(args.output_dir):\n",
    "   os.makedirs(args.output_dir)\n",
    "\n",
    "model_file = f\"{args.output_dir}/{args.name}.pt\"\n",
    "out_file   = f\"{args.output_dir}/{args.name}.json\"\n",
    "\n",
    "if args.save_model:\n",
    "   torch.save(net.state_dict(), model_file)\n",
    "   vprint(f\"Saved model weights into '{model_file}'.\")\n",
    "\n",
    "results_va[\"classification\"][\"num_pos\"] = num_pos_va\n",
    "results_va[\"classification\"][\"num_neg\"] = num_neg_va\n",
    "results_va[\"regression\"][\"num_samples\"] = num_regr_va\n",
    "\n",
    "if results_tr is not None:\n",
    "    results_tr[\"classification\"][\"num_pos\"] = num_pos - num_pos_va\n",
    "    results_tr[\"classification\"][\"num_neg\"] = num_neg - num_neg_va\n",
    "    results_tr[\"regression\"][\"num_samples\"] = num_regr - num_regr_va\n",
    "\n",
    "stats=None\n",
    "if args.normalize_regression == 1 :\n",
    "   stats={}\n",
    "   stats[\"mean\"] = mean_save\n",
    "   stats[\"var\"]  = np.array(var_save)[0]\n",
    "sc.save_results(out_file, args, validation=results_va, training=results_tr, stats=stats)\n",
    "\n",
    "vprint(f\"Saved config and results into '{out_file}'.\\nYou can load the results by:\\n  import sparsechem as sc\\n  res = sc.load_results('{out_file}')\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "943be739",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:52:24.105337Z",
     "start_time": "2022-11-19T21:52:18.804794Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style>\n",
       "    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
       "    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
       "    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
       "    </style>\n",
       "<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>auc_pr</td><td>▁▃▃▃▅▅▆█▆▆▇▇▆▇▅▆█▆▆▇█▇█▇▆▆▇▅▇▅▅▆▅▄▅▄▅▆▇▇</td></tr><tr><td>auc_pr_cal</td><td>▁▄▆▅▆▇▇█▇▇██▇▇▆▇█▇▇▇█▇██▇▇▇▆▇▆▆▇▆▆▆▆▆▇▇▇</td></tr><tr><td>avg_prec_score</td><td>▁▃▄▄▆▆▆█▇▇▇▇▇▇▅▇█▆▆▇█▇█▇▆▆▇▅▇▅▅▆▅▄▅▅▅▆▇▇</td></tr><tr><td>bceloss</td><td>█▅▂▂▂▂▁▁▂▂▂▂▂▃▃▂▂▃▂▂▃▃▃▃▃▄▃▃▃▄▄▃▃▄▄▃▃▃▃▃</td></tr><tr><td>best_accuracy</td><td>▁▄█▅▆▆█████████████▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇</td></tr><tr><td>best_epoch</td><td>▁▁▂▂▂▃▃▃▃▃▃▃▃▃▃▃▃▇▇█████████████████████</td></tr><tr><td>best_roc_auc</td><td>▁▄▇▇▇▇██████████████████████████████████</td></tr><tr><td>f1_max</td><td>▁▄▇▆▆▇▆█▇▇▆▆▇▇▆▆█▆▆▇▇▇▇▆▆▆▇▆▇▇▅▆▆▅▅▆▆▆▆▆</td></tr><tr><td>kappa</td><td>▁▃▆▅▆▆▇▇▆▇▇▆█▇▇▇▇▇▇██▇▇▇█▇▇█▇▇▇▇█▇█▇▇▇██</td></tr><tr><td>kappa_max</td><td>▁▃█▆▇▇▇█▇▇▇▆▇█▇▆█▇▇▇█▇█▆▇▆▇▇▇▇▆▆▆▆▆▇▆▇▆▆</td></tr><tr><td>logloss</td><td>█▅▂▃▂▁▂▁▃▃▃▃▃▄▄▄▄▅▅▄▄▅▄▅▅▆▆▅▆▇▇▆▆▇▆▅▇▆▆▇</td></tr><tr><td>num_tasks_agg</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>num_tasks_total</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>p_f1_max</td><td>▆▂█▆▅▄▆▄▃▅▅▄▅▃▄▄▃▄▄▆▆▃▅▄▇▄▄▅▂▃▂▃▄▁▄▂▂▅▄▅</td></tr><tr><td>p_kappa_max</td><td>▅▄█▆▄▅▇▄▅▅▇▆▄▃▆▆▅▆▅▆▆▃▆▅▆▄▄▃▃▅▄▄▅▁▅▃▃▅▅▄</td></tr><tr><td>roc_auc_score</td><td>▁▄▆▆▇▇▇█▇████▇█████████▇█████▇▇█▇▇▇▇████</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>auc_pr</td><td>0.58707</td></tr><tr><td>auc_pr_cal</td><td>0.44584</td></tr><tr><td>avg_prec_score</td><td>0.59387</td></tr><tr><td>bceloss</td><td>0.36495</td></tr><tr><td>f1_max</td><td>0.61487</td></tr><tr><td>kappa</td><td>0.2739</td></tr><tr><td>kappa_max</td><td>0.44113</td></tr><tr><td>logloss</td><td>0.2214</td></tr><tr><td>num_tasks_agg</td><td>37.0</td></tr><tr><td>num_tasks_total</td><td>72.0</td></tr><tr><td>p_f1_max</td><td>0.2986</td></tr><tr><td>p_kappa_max</td><td>0.36346</td></tr><tr><td>roc_auc_score</td><td>0.80168</td></tr></table><br/></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">1119_1327</strong>: <a href=\"https://wandb.ai/kbardool/SC-cb29-TaskGroups/runs/2i0valte\" target=\"_blank\">https://wandb.ai/kbardool/SC-cb29-TaskGroups/runs/2i0valte</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20221119_132741-2i0valte/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ns.wandb_run.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7143060c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47db9959",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d288b9cd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-19T21:52:45.697222Z",
     "start_time": "2022-11-19T21:52:45.668937Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Best Epoch            49\n",
      " Best Iteration         \n",
      " Best Precision        0.59409\n",
      " Best ROC AUC          0.80473 \n",
      " e                        \n",
      " roc_auc_score         0.8017\n",
      " auc_pr                0.5871\n",
      " avg_prec_score        0.5939\n",
      " f1_max                0.6149\n",
      " p_f1_max              0.2986\n",
      " kappa                 0.2739\n",
      " kappa_max             0.4411\n",
      " p_kappa_max           0.3635\n",
      " bceloss               0.3650\n",
      " auc_pr_cal            0.4458\n",
      " logloss               0.2214\n",
      " num_tasks_total       72.0000\n",
      " num_tasks_agg         37.0000\n"
     ]
    }
   ],
   "source": [
    "print(f\" Best Epoch            {ns.best_epoch}\\n\"\n",
    "      f\" Best Iteration         \\n\"\n",
    "      f\" Best Precision        {ns.best_accuracy:.5f}\\n\"\n",
    "      f\" Best ROC AUC          {ns.best_roc_auc:.5f} \\n\"\n",
    "      f\" e                        \" )\n",
    "\n",
    "for k in results_va['classification_agg'].keys():\n",
    "    print(f\" {k:20s}  {results_va['classification_agg'][k]:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "844ae3ce",
   "metadata": {},
   "outputs": [],
   "source": [
    " Best Epoch            49\n",
    " Best Iteration         \n",
    " Best Precision        0.59409\n",
    " Best ROC AUC          0.80473 \n",
    " e                        \n",
    " roc_auc_score         0.8017\n",
    " auc_pr                0.5871\n",
    " avg_prec_score        0.5939\n",
    " f1_max                0.6149\n",
    " p_f1_max              0.2986\n",
    " kappa                 0.2739\n",
    " kappa_max             0.4411\n",
    " p_kappa_max           0.3635\n",
    " bceloss               0.3650\n",
    " auc_pr_cal            0.4458\n",
    " logloss               0.2214\n",
    " num_tasks_total       72.0000\n",
    " num_tasks_agg         37.0000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "61e959ac",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-18T20:23:13.996228Z",
     "start_time": "2022-11-18T20:23:13.950387Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      roc_auc_score    auc_pr  avg_prec_score    f1_max  p_f1_max     kappa  kappa_max  p_kappa_max   bceloss  auc_pr_cal  num_pos  num_neg\n",
      "task                                                                                                                                       \n",
      "0          0.725183  0.390330        0.397317  0.480818  0.102850  0.190357   0.278974     0.308707  0.525729    0.207653      121      435\n",
      "1          0.796608  0.342250        0.352984  0.505051  0.100710  0.060541   0.457118     0.100710  0.253482    0.388667       45      511\n",
      "2          0.889530  0.302272        0.336348  0.457143  0.047716  0.000000   0.439700     0.047716  0.106344    0.550981       16      542\n",
      "3          0.817331  0.376459        0.382937  0.500000  0.023337  0.000000   0.497297     0.023337  0.055430    0.559964        6      552\n",
      "4          0.703423  0.331365        0.336239  0.398467  0.208077  0.206398   0.281619     0.257336  0.458299    0.233753      115      611\n",
      "...             ...       ...             ...       ...       ...       ...        ...          ...       ...         ...      ...      ...\n",
      "186        0.833653  0.818925        0.820120  0.788462  0.220088  0.486301   0.574594     0.261665  0.529588    0.488045       91      109\n",
      "187        0.933579  0.745060        0.749199  0.750000  0.310593  0.483738   0.679907     0.310593  0.269617    0.585330       41      159\n",
      "188        0.652038  0.807419        0.817432  0.878788  0.926356  0.000000   0.396226     0.928864  0.804994    0.251437       29       11\n",
      "189        0.695157  0.595706        0.608026  0.580645  0.805046  0.000000   0.380805     0.830350  1.115915    0.342184       13       27\n",
      "190        0.434286  0.106570        0.132665  0.260870  0.308717  0.000000   0.081081     0.308717  0.473045    0.085225        5       35\n",
      "\n",
      "[143 rows x 12 columns]\n"
     ]
    }
   ],
   "source": [
    "pd.options.display.width = 150\n",
    "df = results_va['classification']\n",
    "print(df[pd.notna(df.roc_auc_score)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "15ce1b83",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-18T20:23:16.258376Z",
     "start_time": "2022-11-18T20:23:16.216233Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "      roc_auc_score    auc_pr  avg_prec_score    f1_max  p_f1_max     kappa  kappa_max  p_kappa_max   bceloss  auc_pr_cal  num_pos  num_neg\n",
      "task                                                                                                                                       \n",
      "0          0.725183  0.390330        0.397317  0.480818  0.102850  0.190357   0.278974     0.308707  0.525729    0.207653      121      435\n",
      "1          0.796608  0.342250        0.352984  0.505051  0.100710  0.060541   0.457118     0.100710  0.253482    0.388667       45      511\n",
      "2          0.889530  0.302272        0.336348  0.457143  0.047716  0.000000   0.439700     0.047716  0.106344    0.550981       16      542\n",
      "3          0.817331  0.376459        0.382937  0.500000  0.023337  0.000000   0.497297     0.023337  0.055430    0.559964        6      552\n",
      "4          0.703423  0.331365        0.336239  0.398467  0.208077  0.206398   0.281619     0.257336  0.458299    0.233753      115      611\n",
      "5          0.735419  0.158335        0.167101  0.271605  0.107788  0.044187   0.230952     0.107788  0.171308    0.284719       31      693\n",
      "6          0.818820  0.038540        0.048431  0.117647  0.014110  0.000000   0.099520     0.014110  0.055952    0.271981        8      712\n",
      "7          0.888889  0.025334        0.044872  0.133333  0.014359  0.000000   0.127517     0.014359  0.024198    0.317948        3      717\n",
      "8          0.837662  0.911787        0.912916  0.823529  0.125629  0.427136   0.559715     0.579480  0.479435    0.699311       35       22\n",
      "9          0.919355  0.937343        0.938216  0.875000  0.272809  0.597303   0.785176     0.272809  0.425586    0.831775       26       31\n",
      "10         0.924074  0.843786        0.847806  0.800000  0.338502  0.441176   0.759494     0.338502  0.318047    0.776196       12       45\n",
      "11         0.885621  0.352607        0.449074  0.625000  0.015205  0.000000   0.568182     0.015205  0.431435    0.340954        6       51\n",
      "12         0.911644  0.921358        0.921409  0.868932  0.317970  0.633972   0.666058     0.434425  0.381585    0.571508      189      143\n",
      "13         0.865443  0.738376        0.739495  0.741214  0.082370  0.408345   0.558740     0.155663  0.483385    0.428580      124      235\n",
      "14         0.916839  0.546816        0.554777  0.600000  0.077529  0.000000   0.559915     0.077529  0.229247    0.574990       29      300\n",
      "15         0.773700  0.067614        0.131803  0.333333  0.034285  0.000000   0.327886     0.034285  0.033159    0.292493        2      327\n",
      "16         0.793237  0.808430        0.808727  0.759024  0.235681  0.416432   0.433480     0.405970  0.587736    0.409363      423      395\n",
      "17         0.807273  0.658018        0.658755  0.621392  0.155082  0.313191   0.422027     0.161493  0.519729    0.421589      242      582\n",
      "18         0.822606  0.526587        0.529519  0.516432  0.136106  0.321854   0.444744     0.136106  0.350919    0.466562      110      714\n",
      "19         0.863912  0.482247        0.488753  0.558140  0.116392  0.072072   0.534159     0.116392  0.180729    0.585700       48      776\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(results_va['classification'][0:20])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4bbb2748",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-18T20:23:21.273959Z",
     "start_time": "2022-11-18T20:23:21.236987Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "dict_keys(['classification', 'classification_agg', 'regression', 'regression_agg'])\n",
      "roc_auc_score        0.758299\n",
      "auc_pr               0.645717\n",
      "avg_prec_score       0.653529\n",
      "f1_max               0.687732\n",
      "p_f1_max             0.283795\n",
      "kappa                0.279043\n",
      "kappa_max            0.435864\n",
      "p_kappa_max          0.393663\n",
      "bceloss              0.528878\n",
      "auc_pr_cal           0.395667\n",
      "logloss              0.354635\n",
      "num_tasks_total    192.000000\n",
      "num_tasks_agg       71.000000\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(results_va.keys())\n",
    "pp.pprint(results_va['classification_agg'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "ebf2a3d4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-18T20:23:21.962993Z",
     "start_time": "2022-11-18T20:23:21.914621Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      roc_auc_score  auc_pr  avg_prec_score  f1_max  p_f1_max  kappa  kappa_max  p_kappa_max  bceloss  auc_pr_cal  num_pos  num_neg\n",
      "task                                                                                                                               \n",
      "27              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      113\n",
      "31              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      104\n",
      "35              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       58\n",
      "48              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      108\n",
      "49              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      108\n",
      "50              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      108\n",
      "51              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      108\n",
      "57              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      165\n",
      "58              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      165\n",
      "59              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      165\n",
      "63              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       25\n",
      "66              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       96\n",
      "67              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       96\n",
      "74              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       23\n",
      "75              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       23\n",
      "78              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       90\n",
      "79              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       90\n",
      "83              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       36\n",
      "87              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       27\n",
      "90              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       87\n",
      "91              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       87\n",
      "94              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       99\n",
      "95              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       99\n",
      "96              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       28\n",
      "97              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       28\n",
      "98              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       28\n",
      "99              NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       28\n",
      "103             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       20\n",
      "107             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       79\n",
      "112             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN       41        0\n",
      "123             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       81\n",
      "131             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0      122\n",
      "136             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN       37        0\n",
      "141             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       20\n",
      "142             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       20\n",
      "143             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       20\n",
      "146             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       68\n",
      "147             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       68\n",
      "156             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        1        0\n",
      "157             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        1        0\n",
      "158             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        1        0\n",
      "159             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        1        0\n",
      "163             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       57\n",
      "171             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       35\n",
      "175             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       16\n",
      "177             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       19\n",
      "178             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       19\n",
      "179             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       19\n",
      "191             NaN     NaN             NaN     NaN       NaN    NaN        NaN          NaN      NaN         NaN        0       40\n",
      "      roc_auc_score    auc_pr  avg_prec_score    f1_max  p_f1_max     kappa  kappa_max  p_kappa_max   bceloss  auc_pr_cal  num_pos  num_neg\n",
      "task                                                                                                                                       \n",
      "0          0.725183  0.390330        0.397317  0.480818  0.102850  0.190357   0.278974     0.308707  0.525729    0.207653      121      435\n",
      "1          0.796608  0.342250        0.352984  0.505051  0.100710  0.060541   0.457118     0.100710  0.253482    0.388667       45      511\n",
      "2          0.889530  0.302272        0.336348  0.457143  0.047716  0.000000   0.439700     0.047716  0.106344    0.550981       16      542\n",
      "3          0.817331  0.376459        0.382937  0.500000  0.023337  0.000000   0.497297     0.023337  0.055430    0.559964        6      552\n",
      "4          0.703423  0.331365        0.336239  0.398467  0.208077  0.206398   0.281619     0.257336  0.458299    0.233753      115      611\n",
      "...             ...       ...             ...       ...       ...       ...        ...          ...       ...         ...      ...      ...\n",
      "186        0.833653  0.818925        0.820120  0.788462  0.220088  0.486301   0.574594     0.261665  0.529588    0.488045       91      109\n",
      "187        0.933579  0.745060        0.749199  0.750000  0.310593  0.483738   0.679907     0.310593  0.269617    0.585330       41      159\n",
      "188        0.652038  0.807419        0.817432  0.878788  0.926356  0.000000   0.396226     0.928864  0.804994    0.251437       29       11\n",
      "189        0.695157  0.595706        0.608026  0.580645  0.805046  0.000000   0.380805     0.830350  1.115915    0.342184       13       27\n",
      "190        0.434286  0.106570        0.132665  0.260870  0.308717  0.000000   0.081081     0.308717  0.473045    0.085225        5       35\n",
      "\n",
      "[143 rows x 12 columns]\n"
     ]
    }
   ],
   "source": [
    "print(df[pd.isna(df.roc_auc_score)])\n",
    "print(df[pd.notna(df.roc_auc_score)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "a56ba82a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-11-18T20:23:22.945368Z",
     "start_time": "2022-11-18T20:23:22.910760Z"
    }
   },
   "outputs": [],
   "source": [
    "# for p in net.parameters():\n",
    "#     print(p.shape, p.numel())\n",
    "\n",
    "# print()\n",
    "# print(args.lr_steps, args.lr_alpha, args.weight_decay)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3de95601",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Results "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a95fe27d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T15:17:56.503884Z",
     "start_time": "2022-04-28T15:17:56.464295Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "      roc_auc_score    auc_pr  avg_prec_score    f1_max  p_f1_max     kappa  \\\n",
      "task                                                                          \n",
      "0          0.991560  0.998782        0.998752  0.982301  0.980404  0.786753   \n",
      "1               NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "2               NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "3               NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "4               NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "5          0.944444  0.996995        0.997076  0.972973  0.562853  0.000000   \n",
      "6          1.000000  1.000000        1.000000  1.000000  0.973055  0.797642   \n",
      "7          0.666667  0.461111        0.588889  0.750000  0.763078  0.000000   \n",
      "8          1.000000  1.000000        1.000000  1.000000  0.987239  0.000000   \n",
      "9          1.000000  1.000000        1.000000  1.000000  0.998716  0.660870   \n",
      "10              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "11              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "12              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "13              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "14         0.993333  0.999823        0.999825  0.993377  0.959745  0.000000   \n",
      "15              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "16              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "17              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "18              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "19         1.000000  1.000000        1.000000  1.000000  0.957944  0.000000   \n",
      "20              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "21              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "22              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "23              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "24              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "25              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "26              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "27              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "28         0.915152  0.997941        0.997946  0.996979  0.875137  0.797101   \n",
      "29              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "30              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "31              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "32              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "33              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "34              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "35         1.000000  1.000000        1.000000  1.000000  0.979840  1.000000   \n",
      "36         0.750000  0.766667        0.804167  0.888889  0.083961  0.222222   \n",
      "37         0.619048  0.845833        0.857143  0.823529  0.287464  0.347826   \n",
      "38         0.378378  0.978327        0.978592  0.991071  0.000882 -0.011940   \n",
      "39              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "40              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "41         1.000000  1.000000        1.000000  1.000000  0.998889  0.000000   \n",
      "42              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "43         0.474227  0.992424        0.992485  0.994872  0.995585  0.000000   \n",
      "44         0.993971  0.999149        0.999152  0.992248  0.818924  0.829506   \n",
      "45              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "46         0.750000  0.939224        0.943012  0.952381  0.985955  0.000000   \n",
      "47              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "48         0.888889  0.741100        0.728505  0.750000  0.582740  0.727141   \n",
      "49              NaN       NaN             NaN       NaN       NaN       NaN   \n",
      "\n",
      "      kappa_max  p_kappa_max   bceloss  auc_pr_cal  num_pos  num_neg  \n",
      "task                                                                  \n",
      "0      0.877266     0.980404  0.099664    0.976466      115       17  \n",
      "1           NaN          NaN       NaN         NaN       31        0  \n",
      "2           NaN          NaN       NaN         NaN       17        0  \n",
      "3           NaN          NaN       NaN         NaN       54        0  \n",
      "4           NaN          NaN       NaN         NaN        7        0  \n",
      "5      0.641509     0.986980  0.111792    0.949860       18        1  \n",
      "6      1.000000     0.973055  0.016514    1.000000      203        3  \n",
      "7      0.461538     0.763078  1.387727    0.132914        3        4  \n",
      "8      1.000000     0.987239  0.094224    1.000000       55        2  \n",
      "9      1.000000     0.998716  0.016673    1.000000       76        2  \n",
      "10          NaN          NaN       NaN         NaN       63        0  \n",
      "11          NaN          NaN       NaN         NaN        1        0  \n",
      "12          NaN          NaN       NaN         NaN       42        0  \n",
      "13          NaN          NaN       NaN         NaN        0        1  \n",
      "14     0.793566     0.995055  0.069398    0.989078       75        2  \n",
      "15          NaN          NaN       NaN         NaN       36        0  \n",
      "16          NaN          NaN       NaN         NaN        0        0  \n",
      "17          NaN          NaN       NaN         NaN        1        0  \n",
      "18          NaN          NaN       NaN         NaN       43        0  \n",
      "19     1.000000     0.957944  1.159957    1.000000        2        2  \n",
      "20          NaN          NaN       NaN         NaN        7        0  \n",
      "21          NaN          NaN       NaN         NaN        7        0  \n",
      "22          NaN          NaN       NaN         NaN        0       13  \n",
      "23          NaN          NaN       NaN         NaN        3        0  \n",
      "24          NaN          NaN       NaN         NaN        0        0  \n",
      "25          NaN          NaN       NaN         NaN       21        0  \n",
      "26          NaN          NaN       NaN         NaN        1        0  \n",
      "27          NaN          NaN       NaN         NaN       37        0  \n",
      "28     0.797101     0.875137  0.042430    0.676589      330        6  \n",
      "29          NaN          NaN       NaN         NaN       11        0  \n",
      "30          NaN          NaN       NaN         NaN       86        0  \n",
      "31          NaN          NaN       NaN         NaN        0       30  \n",
      "32          NaN          NaN       NaN         NaN        0        0  \n",
      "33          NaN          NaN       NaN         NaN      142        0  \n",
      "34          NaN          NaN       NaN         NaN       27        0  \n",
      "35     1.000000     0.979840  0.006316    1.000000       25        3  \n",
      "36     0.695652     0.083961  1.078320    0.376580        4        3  \n",
      "37     0.347826     0.812577  0.621829    0.498911        7        3  \n",
      "38     0.010991     0.999271  0.210472    0.203723      111        2  \n",
      "39          NaN          NaN       NaN         NaN       12        0  \n",
      "40          NaN          NaN       NaN         NaN        7        0  \n",
      "41     1.000000     0.998889  0.063473    1.000000       79        1  \n",
      "42          NaN          NaN       NaN         NaN        0        1  \n",
      "43     0.018075     0.999811  0.087830    0.513910       97        1  \n",
      "44     0.936693     0.818924  0.067191    0.970516      129       18  \n",
      "45          NaN          NaN       NaN         NaN       15        0  \n",
      "46     0.625000     0.985955  0.576575    0.571078       10        2  \n",
      "47          NaN          NaN       NaN         NaN        9        0  \n",
      "48     0.727141     0.582740  0.222837    0.745717        9       85  \n",
      "49          NaN          NaN       NaN         NaN       38        0  \n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(results_va['classification'][0:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "deca5b86",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T16:20:53.027104Z",
     "start_time": "2022-04-28T16:20:52.995717Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2445\n",
      "18704\n",
      "505\n",
      "3804\n"
     ]
    }
   ],
   "source": [
    "print( num_neg.sum())\n",
    "print( num_pos.sum())\n",
    "print( num_neg_va.sum())\n",
    "print( num_pos_va.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "545b94be",
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for i in zip(dldrs.valset.num_pos, dldrs.valset.num_neg):\n",
    "    print(f\" {i[0]:4d}  {i[1]:4d}\")\n",
    "for i in zip(dldrs.valset.num_pos, dldrs.valset.num_neg):\n",
    "    print(f\" {i[0]:4d}  {i[1]:4d}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3fa5e170",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T15:18:02.976078Z",
     "start_time": "2022-04-28T15:18:02.936187Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "roc_auc_score        0.900757\n",
      "auc_pr               0.943594\n",
      "avg_prec_score       0.945930\n",
      "f1_max               0.938011\n",
      "p_f1_max             0.723872\n",
      "kappa                0.611823\n",
      "kappa_max            0.803714\n",
      "p_kappa_max          0.826011\n",
      "bceloss              0.296447\n",
      "auc_pr_cal           0.827893\n",
      "logloss              0.087415\n",
      "num_tasks_total    100.000000\n",
      "num_tasks_agg       20.000000\n",
      "dtype: float64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# pp.pprint(results_va)\n",
    "print()\n",
    "\n",
    "pp.pprint(results_va['classification_agg'])\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "00b24bc3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T16:12:52.731012Z",
     "start_time": "2022-04-28T16:12:52.668446Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[115  31  17  54   7  18 203   3  55  76  63   1  42   0  75  36   0   1  43   2   7   7   0   3   0  21   1  37 330  11  86   0   0 142  27  25   4\n",
      "   7 111  12   7  79   0  97 129  15  10   9   9  38 272   1   2   0 285  63 185   1   0   4  19  21  13   0  18  12   0   0  93  55   1  73  29  41\n",
      "  18   0  46  37  14   0  40  18   6   8  10 330   6   4   1   0   0   0   8   0   0   0   3   0   1   0]\n",
      "[17  0  0  0  0  1  3  4  2  2  0  0  0  1  2  0  0  0  0  2  0  0 13  0  0  0  0  0  6  0  0 30  0  0  0  3  3  3  2  0  0  1  1  1 18  0  2  0 85\n",
      "  0  0  2 13 47  1 11  0  0  0  2  0  1  1  1  3  0  0  0  0  0 18  0  9  3  0  4  1  0 23 11  6  0  5  0  0 46  4  4 73  2  0  7  1  0  0  0  0  0\n",
      "  4  0]\n",
      "[]\n",
      "  115    17    trianing:  884     83\n",
      "   31     0    trianing:   59      0\n",
      "   17     0    trianing:   59      0\n",
      "   54     0    trianing:  284      0\n",
      "    7     0    trianing:   55      0\n",
      "   18     1    trianing:   43      3\n",
      "  203     3    trianing:  749     29\n",
      "    3     4    trianing:   14      7\n",
      "   55     2    trianing:  292      9\n",
      "   76     2    trianing:  342     11\n",
      "   63     0    trianing:  423      0\n",
      "    1     0    trianing:   13      0\n",
      "   42     0    trianing:  119      0\n",
      "    0     1    trianing:    1     12\n",
      "   75     2    trianing:  281      3\n",
      "   36     0    trianing:  213      0\n",
      "    0     0    trianing:    0      0\n",
      "    1     0    trianing:   11      0\n",
      "   43     0    trianing:  190      0\n",
      "    2     2    trianing:   27      9\n",
      "    7     0    trianing:   46      0\n",
      "    7     0    trianing:   36     13\n",
      "    0    13    trianing:   16     21\n",
      "    3     0    trianing:   43      0\n",
      "    0     0    trianing:    9      0\n",
      "   21     0    trianing:   70      0\n",
      "    1     0    trianing:    7      0\n",
      "   37     0    trianing:  163      0\n",
      "  330     6    trianing: 1399     45\n",
      "   11     0    trianing:   35      0\n",
      "   86     0    trianing:  447      0\n",
      "    0    30    trianing:    1    143\n",
      "    0     0    trianing:    0      0\n",
      "  142     0    trianing:  675      9\n",
      "   27     0    trianing:  327      0\n",
      "   25     3    trianing:  142     56\n",
      "    4     3    trianing:    6      9\n",
      "    7     3    trianing:   31     18\n",
      "  111     2    trianing:  552     17\n",
      "   12     0    trianing:   52      0\n",
      "    7     0    trianing:   27      0\n",
      "   79     1    trianing:  325      1\n",
      "    0     1    trianing:    1      1\n",
      "   97     1    trianing:  350      1\n",
      "  129    18    trianing:  535     72\n",
      "   15     0    trianing:   72      0\n",
      "   10     2    trianing:   94     10\n",
      "    9     0    trianing:   36      0\n",
      "    9    85    trianing:   55    280\n",
      "   38     0    trianing:  259      2\n",
      "  272     0    trianing: 1366      0\n",
      "    1     2    trianing:    3      2\n",
      "    2    13    trianing:   24     48\n",
      "    0    47    trianing:    9    245\n",
      "  285     1    trianing: 1408     17\n",
      "   63    11    trianing:  148     77\n",
      "  185     0    trianing:  971      4\n",
      "    1     0    trianing:  113      0\n",
      "    0     0    trianing:   31      0\n",
      "    4     2    trianing:   55      9\n",
      "   19     0    trianing:   70      0\n",
      "   21     1    trianing:   74      5\n",
      "   13     1    trianing:   63      3\n",
      "    0     1    trianing:    0      2\n",
      "   18     3    trianing:   86     57\n",
      "   12     0    trianing:   58      1\n",
      "    0     0    trianing:    2      0\n",
      "    0     0    trianing:    0      1\n",
      "   93     0    trianing:  465      5\n",
      "   55     0    trianing:  273      2\n",
      "    1    18    trianing:    9     44\n",
      "   73     0    trianing:  317      2\n",
      "   29     9    trianing:   91     64\n",
      "   41     3    trianing:  146     11\n",
      "   18     0    trianing:  199      1\n",
      "    0     4    trianing:    0     24\n",
      "   46     1    trianing:  251      3\n",
      "   37     0    trianing:  232      0\n",
      "   14    23    trianing:   62    101\n",
      "    0    11    trianing:   14     21\n",
      "   40     6    trianing:  230     38\n",
      "   18     0    trianing:   79      0\n",
      "    6     5    trianing:   30     19\n",
      "    8     0    trianing:   41      0\n",
      "   10     0    trianing:   32      1\n",
      "  330    46    trianing: 1610    220\n",
      "    6     4    trianing:   74     16\n",
      "    4     4    trianing:   16     13\n",
      "    1    73    trianing:   22    400\n",
      "    0     2    trianing:    0     32\n",
      "    0     0    trianing:    0      0\n",
      "    0     7    trianing:    0     27\n",
      "    8     1    trianing:  102      1\n",
      "    0     0    trianing:    1     17\n",
      "    0     0    trianing:    0      0\n",
      "    0     0    trianing:   32      0\n",
      "    3     0    trianing:   14      2\n",
      "    0     0    trianing:    0      0\n",
      "    1     4    trianing:   10     32\n",
      "    0     0    trianing:    1     14\n"
     ]
    }
   ],
   "source": [
    "print(num_pos_va)\n",
    "print(num_neg_va)\n",
    "print(num_regr_va)\n",
    "for i in zip(num_pos_va, num_neg_va, num_pos, num_neg):\n",
    "    print(f\" {i[0]:4d}  {i[1]:4d}    trianing: {i[2]:4d}   {i[3]:4d}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "40b7bd71",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T15:22:06.177783Z",
     "start_time": "2022-04-28T15:22:06.127520Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "roc_auc_score      0.874262\n",
       "auc_pr             0.922464\n",
       "avg_prec_score     0.932781\n",
       "f1_max             0.938123\n",
       "p_f1_max           0.746278\n",
       "kappa              0.400306\n",
       "kappa_max          0.754262\n",
       "p_kappa_max        0.830485\n",
       "bceloss            0.385833\n",
       "auc_pr_cal         0.803581\n",
       "num_pos           58.512821\n",
       "num_neg            9.948718\n",
       "dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[pd.notna(df.roc_auc_score)].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b482ac5a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-28T15:22:26.189727Z",
     "start_time": "2022-04-28T15:22:26.149010Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "del net"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fa96df3",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Misc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ff228568",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-13T07:37:52.305993Z",
     "start_time": "2022-04-13T07:37:52.296939Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import wandb\n",
    "def restart_wandb(exp_id, exp_name, project_name, resume = \"allow\" ):\n",
    "    print(exp_id, exp_name, project_name) \n",
    "    wandb_run = wandb.init(project = project_name, \n",
    "                                     entity  = \"kbardool\", \n",
    "                                     id      = exp_id, \n",
    "                                     name    = exp_name,\n",
    "                                     resume=resume )\n",
    "    \n",
    "    print(f\" PROJECT NAME: {wandb_run.project}\\n\"\n",
    "          f\" RUN ID      : {wandb_run.id} \\n\"\n",
    "          f\" RUN NAME    : {wandb_run.name}\")     \n",
    " \n",
    "    return wandb_run \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "51adf2f8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-13T07:37:58.779180Z",
     "start_time": "2022-04-13T07:37:53.070725Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d2rw3bdq 0413_0509 SparseChem-Mini\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Finishing last run (ID:d2rw3bdq) before initializing another..."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Waiting for W&B process to finish... <strong style=\"color:green\">(success).</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8471011283e94d06a59a2d75df43c046",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(Label(value='0.000 MB of 0.000 MB uploaded (0.000 MB deduped)\\r'), FloatProgress(value=1.0, max…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
      "WARNING:\n",
      "\n",
      "You should always run with libnvidia-ml.so that is installed with your\n",
      "NVIDIA Display Driver. By default it's installed in /usr/lib and /usr/lib64.\n",
      "libnvidia-ml.so in GDK package is a stub library that is attached only for\n",
      "build purposes (e.g. machine that you build your application doesn't have\n",
      "to have Display Driver installed).\n",
      "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n",
      "Linked to libnvidia-ml library at wrong path : /usr/lib64/libnvidia-ml.so.1\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Synced <strong style=\"color:#cdcd00\">0413_0509</strong>: <a href=\"https://wandb.ai/kbardool/SparseChem-Mini/runs/d2rw3bdq\" target=\"_blank\">https://wandb.ai/kbardool/SparseChem-Mini/runs/d2rw3bdq</a><br/>Synced 3 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Find logs at: <code>./wandb/run-20220413_093712-d2rw3bdq/logs</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Successfully finished last run (ID:d2rw3bdq). Initializing new run:<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.14 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.11"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/vsc-hard-mounts/leuven-data/326/vsc32647/projs/SparseChem/wandb/run-20220413_093753-d2rw3bdq</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Resuming run <strong><a href=\"https://wandb.ai/kbardool/SparseChem-Mini/runs/d2rw3bdq\" target=\"_blank\">0413_0509</a></strong> to <a href=\"https://wandb.ai/kbardool/SparseChem-Mini\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " PROJECT NAME: SparseChem-Mini\n",
      " RUN ID      : d2rw3bdq \n",
      " RUN NAME    : 0413_0509\n"
     ]
    }
   ],
   "source": [
    "run = restart_wandb(\"d2rw3bdq\",\"0413_0509\",\"SparseChem-Mini\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "dcc605ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-13T07:49:06.657692Z",
     "start_time": "2022-04-13T07:49:06.645775Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'run' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m/tmp/a5e50704be/ipykernel_31013/1322073845.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'run' is not defined"
     ]
    }
   ],
   "source": [
    "print(run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "271944b2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-13T07:48:13.315835Z",
     "start_time": "2022-04-13T07:48:13.311428Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "run.finish()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "b561a453",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-13T07:49:03.347430Z",
     "start_time": "2022-04-13T07:49:03.342915Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "del run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ffe3f20",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-10T18:02:22.065859Z",
     "start_time": "2022-04-10T18:02:21.337434Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# cmd = (\n",
    "#   f\" --x       /home/kbardool/kusanagi/MLDatasets/chembl_23mini_synthetic/chembl_23mini_x.npy \" +\n",
    "#   f\" --y_class /home/kbardool/kusanagi/MLDatasets/chembl_23mini_synthetic/chembl_23mini_adashare_y_all_bin_sparse.npy \" +\n",
    "#   f\" --folding /home/kbardool/kusanagi/MLDatasets/chembl_23mini_synthetic/chembl_23mini_folds.npy \" +\n",
    "#   f\" --output_dir {output_dir}\" +    \n",
    "#   f\" --fold_va           0 \" +\n",
    "#   f\" --batch_ratio    0.02 \" +\n",
    "#   f\" --hidden_sizes   25 25 25 25 25 25 \" +\n",
    "#   f\" --dropouts_trunk  0  0  0  0  0  0 \" +\n",
    "#   f\" --weight_decay   1e-4 \" \n",
    "#   f\" --epochs           40 \" +\n",
    "#   f\" --lr             1e-3 \" +\n",
    "#   f\" --lr_steps         10 \" +\n",
    "#   f\" --lr_alpha        0.3\" \n",
    "# )\n",
    "\n",
    "# cmd = (\n",
    "#   f\" --x       {data_dir}/chembl_23mini_x.npy \" +\n",
    "#   f\" --y_class {data_dir}/chembl_23mini_adashare_y_all_bin_sparse.npy \" +\n",
    "#   f\" --folding {data_dir}/chembl_23mini_folds.npy \" +\n",
    "#   f\" --output_dir {output_dir}\" +    \n",
    "#   f\" --fold_va            0 \" +\n",
    "#   f\" --batch_ratio     0.02 \" +\n",
    "#   f\" --hidden_sizes   40 40 \" +\n",
    "#   f\" --dropouts_trunk  0  0 \" +\n",
    "#   f\" --weight_decay   1e-4 \" +\n",
    "#   f\" --epochs           20 \" +\n",
    "#   f\" --lr             1e-3 \" +\n",
    "#   f\" --lr_steps         10 \" +\n",
    "#   f\" --lr_alpha        0.3 \" \n",
    "# )\n",
    "\n",
    "#   f\" --hidden_sizes   400 400 \" +\n",
    "#   f\" --last_dropout   0.2 \" +\n",
    "#   f\" --middle_dropout 0.2 \" +\n",
    "#   f\" --x       ./{data_dir}/chembl_23_x.mtx \" +\n",
    "#   f\" --y_class ./{data_dir}/chembl_23_y.mtx \" +\n",
    "#   f\" --folding ./{data_dir}/folding_hier_0.6.npy \" +\n",
    "\n",
    "#### copied from SparseChemDev \n",
    "\n",
    "# cmd = (\n",
    "#         f\" --x       ./{data_dir}/chembl_23mini_x.npy\" +\n",
    "#         f\" --y_class ./{data_dir}/chembl_23mini_y.npy\" +\n",
    "#         f\" --folding ./{data_dir}/chembl_23mini_folds.npy\" +\n",
    "#         f\" --hidden_sizes 20 30 40 \" +  \n",
    "#         f\" --output_dir {output_dir}\" +\n",
    "#         f\" --batch_ratio 0.1\" +\n",
    "#         f\" --epochs 2\" +\n",
    "#         f\" --lr 1e-3\" +\n",
    "#         f\" --lr_steps 1\" +\n",
    "#         f\" --dev {dev}\" +\n",
    "#         f\" --verbose 1\")\n",
    "#         f\" --input_size_freq  40\"\n",
    "#         f\" --tail_hidden_size  10\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "43c68af3",
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# data_dir=\"chembl23_data\"\n",
    "# data_dir=\"chembl23_run_01152022\"\n",
    "# rstr = \"synthetic_data_model\" ##random_str(12)\n",
    "# rstr = \"synthetic_data_model_03042022\" ##random_str(12)\n",
    "# output_dir = f\"./models-{rstr}/\"\n",
    "# output_dir = f\"./{data_dir}/models-{rstr}/\"\n",
    "# output dir kbardool/kusanagi/experiments/SparseChem/0116_0843\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1dd7fea2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-04-10T17:59:54.268012Z",
     "start_time": "2022-04-10T17:59:54.197660Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# dev = \"gpu\" \n",
    "# data_dir=\"chembl23_data\"\n",
    "# data_dir=\"chembl23_run_01152022\"\n",
    "# data_dir = \"/home/kbardool/kusanagi/MLDatasets/chembl_23mini_synthetic\"\n",
    "\n",
    "# rm_output=False\n",
    "\n",
    "# rstr = datetime.now().strftime(\"%m%d_%H%M\")\n",
    "# rstr = \"synthetic_data_model\" ##random_str(12)\n",
    "# rstr = \"synthetic_data_model_03042022\" ##random_str(12)\n",
    "\n",
    "# output_dir = f\"./models-{rstr}/\"\n",
    "# output_dir = f\"./{data_dir}/models-{rstr}/\"\n",
    "\n",
    "# output dir kbardool/kusanagi/experiments/SparseChem/0116_0843\n",
    "# output_dir = f\"/home/kbardool/kusanagi/experiments/SparseChem/{rstr}\"\n",
    "# print(output_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pyt-gpu]",
   "language": "python",
   "name": "conda-env-pyt-gpu-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}