Discovering small-molecule senolytics with deep neural networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This Jupyter notebook accompanies the paper, \"Discoverying senolytics with deep learning\". It reproduces the Chemprop model training and prediction steps used in the paper, thereby providing an integrated platform for interested readers to develop and apply their own Chemprop models to senolytic compound discovery.\n",
    "\n",
    "Key files:\n",
    "- requirements.txt: A Python requirements file detailing all the package dependencies needed to successfully execute all commands in this notebook.\n",
    "\n",
    "- train.csv: A CSV file containing the SMILES strings and activity values of compounds in the training set. The default file provided is the training set of 2,352 compounds described in the main text, with activity values defined by the cell viability criteria described in the main text. There are a total of 45 active (ACTIVITY=1) compounds; the remaining compounds are inactive (ACTIVITY=0).\n",
    "\n",
    "- hyperparameters.json: A JSON file containing key hyperparameters specifying the architecture of the Chemprop model used. By default, the parameters used are: depth=2, dropout=0.0, ffn_num_layers=3, and hidden_size=600, as detailed further in the Methods section of the main text.\n",
    "\n",
    "- test.csv: A CSV file containing the SMILES strings of compounds for which Chemprop predictions of senolytic activity will be made. The default file provided is the set of 266 curated and experimentally tested compounds in our study, a subset of the 804,959 compounds described in the main text; this smaller subset was provided to facilitate evaluation for setups with less computational power. For the full set of compounds computationally evaluated in the paper, please refer to Supplementary Dataset 2. \n",
    "\n",
    "For questions or comments about this notebook, please reach out to felixjwong@gmail.com."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import glob\n",
    "import numpy\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds in training set: 2352 2352\n"
     ]
    }
   ],
   "source": [
    "# Check the number of compounds in the training set\n",
    "TRAIN_SMILES=[];\n",
    "TRAIN_ACTIVITY=[];\n",
    "fopen = open(\"./notebook_files/train.csv\");\n",
    "for line in fopen.readlines()[1:]:\n",
    "    y = [value for value in line.split('\\n')]\n",
    "    y = y[0].split(\",\")\n",
    "    try:\n",
    "        TRAIN_SMILES.append(y[0])\n",
    "        TRAIN_ACTIVITY.append(y[1])\n",
    "    except:\n",
    "        x = 1;\n",
    "fopen.close()\n",
    "print(\"Number of compounds in training set:\" ,len(TRAIN_SMILES),len(TRAIN_ACTIVITY))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "depth 2\n",
      "dropout 0.0\n",
      "ffn_num_layers 3\n",
      "hidden_size 600\n"
     ]
    }
   ],
   "source": [
    "# Check the Chemprop model hyperparameters\n",
    "f = open('notebook_files/hyperparameters.json')\n",
    "data = json.load(f)\n",
    "for i in data:\n",
    "    print(i,data[i])\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chemprop_train --data_path notebook_files/train.csv --dataset_type classification --config_path notebook_files/hyperparameters.json --save_dir \"notebook_files/checkpoint\" --ensemble_size 1 --features_generator rdkit_2d_normalized --no_features_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "Command line\n",
      "python /Users/felix/opt/anaconda3/envs/chemprop/bin/chemprop_train --data_path notebook_files/train.csv --dataset_type classification --config_path notebook_files/hyperparameters.json --save_dir notebook_files/checkpoint --ensemble_size 1 --features_generator rdkit_2d_normalized --no_features_scaling\n",
      "Args\n",
      "{'activation': 'ReLU',\n",
      " 'aggregation': 'mean',\n",
      " 'aggregation_norm': 100,\n",
      " 'alternative_loss_function': None,\n",
      " 'atom_descriptor_scaling': True,\n",
      " 'atom_descriptors': None,\n",
      " 'atom_descriptors_path': None,\n",
      " 'atom_descriptors_size': 0,\n",
      " 'atom_features_size': 0,\n",
      " 'atom_messages': False,\n",
      " 'batch_size': 50,\n",
      " 'bias': False,\n",
      " 'bond_feature_scaling': True,\n",
      " 'bond_features_path': None,\n",
      " 'bond_features_size': 0,\n",
      " 'cache_cutoff': 10000,\n",
      " 'checkpoint_dir': None,\n",
      " 'checkpoint_frzn': None,\n",
      " 'checkpoint_path': None,\n",
      " 'checkpoint_paths': None,\n",
      " 'class_balance': False,\n",
      " 'config_path': 'notebook_files/hyperparameters.json',\n",
      " 'crossval_index_dir': None,\n",
      " 'crossval_index_file': None,\n",
      " 'crossval_index_sets': None,\n",
      " 'cuda': False,\n",
      " 'data_path': 'notebook_files/train.csv',\n",
      " 'data_weights_path': None,\n",
      " 'dataset_type': 'classification',\n",
      " 'depth': 2,\n",
      " 'device': device(type='cpu'),\n",
      " 'dropout': 0.0,\n",
      " 'empty_cache': False,\n",
      " 'ensemble_size': 1,\n",
      " 'epochs': 30,\n",
      " 'explicit_h': False,\n",
      " 'extra_metrics': [],\n",
      " 'features_generator': ['rdkit_2d_normalized'],\n",
      " 'features_only': False,\n",
      " 'features_path': None,\n",
      " 'features_scaling': False,\n",
      " 'features_size': None,\n",
      " 'ffn_hidden_size': 600,\n",
      " 'ffn_num_layers': 3,\n",
      " 'final_lr': 0.0001,\n",
      " 'folds_file': None,\n",
      " 'freeze_first_only': False,\n",
      " 'frzn_ffn_layers': 0,\n",
      " 'gpu': None,\n",
      " 'grad_clip': None,\n",
      " 'hidden_size': 600,\n",
      " 'ignore_columns': None,\n",
      " 'init_lr': 0.0001,\n",
      " 'log_frequency': 10,\n",
      " 'max_data_size': None,\n",
      " 'max_lr': 0.001,\n",
      " 'metric': 'auc',\n",
      " 'metrics': ['auc'],\n",
      " 'minimize_score': False,\n",
      " 'mpn_shared': False,\n",
      " 'multiclass_num_classes': 3,\n",
      " 'no_atom_descriptor_scaling': False,\n",
      " 'no_bond_features_scaling': False,\n",
      " 'no_cache_mol': False,\n",
      " 'no_cuda': False,\n",
      " 'no_features_scaling': True,\n",
      " 'num_folds': 1,\n",
      " 'num_lrs': 1,\n",
      " 'num_tasks': 1,\n",
      " 'num_workers': 8,\n",
      " 'number_of_molecules': 1,\n",
      " 'overwrite_default_atom_features': False,\n",
      " 'overwrite_default_bond_features': False,\n",
      " 'phase_features_path': None,\n",
      " 'pytorch_seed': 0,\n",
      " 'quiet': False,\n",
      " 'reaction': False,\n",
      " 'reaction_mode': 'reac_diff',\n",
      " 'resume_experiment': False,\n",
      " 'save_dir': 'notebook_files/checkpoint',\n",
      " 'save_preds': False,\n",
      " 'save_smiles_splits': False,\n",
      " 'seed': 0,\n",
      " 'separate_test_atom_descriptors_path': None,\n",
      " 'separate_test_bond_features_path': None,\n",
      " 'separate_test_features_path': None,\n",
      " 'separate_test_path': None,\n",
      " 'separate_test_phase_features_path': None,\n",
      " 'separate_val_atom_descriptors_path': None,\n",
      " 'separate_val_bond_features_path': None,\n",
      " 'separate_val_features_path': None,\n",
      " 'separate_val_path': None,\n",
      " 'separate_val_phase_features_path': None,\n",
      " 'show_individual_scores': False,\n",
      " 'smiles_columns': ['\\ufeffSMILES'],\n",
      " 'spectra_activation': 'exp',\n",
      " 'spectra_phase_mask_path': None,\n",
      " 'spectra_target_floor': 1e-08,\n",
      " 'split_sizes': (0.8, 0.1, 0.1),\n",
      " 'split_type': 'random',\n",
      " 'target_columns': None,\n",
      " 'target_weights': None,\n",
      " 'task_names': ['SENOLYTIC'],\n",
      " 'test': False,\n",
      " 'test_fold_index': None,\n",
      " 'train_data_size': None,\n",
      " 'undirected': False,\n",
      " 'use_input_features': True,\n",
      " 'val_fold_index': None,\n",
      " 'warmup_epochs': 2.0}\n",
      "Setting molecule featurization parameters to default.\n",
      "Loading data\n",
      "2352it [00:00, 41020.77it/s]\n",
      "100%|██████████| 2352/2352 [02:23<00:00, 16.41it/s]\n",
      "100%|██████████| 2352/2352 [00:00<00:00, 135342.82it/s]\n",
      "Number of tasks = 1\n",
      "Fold 0\n",
      "Splitting data with seed 0\n",
      "Class sizes\n",
      "SENOLYTIC 0: 98.09%, 1: 1.91%\n",
      "Total size = 2,352 | train size = 1,881 | val size = 235 | test size = 236\n",
      "Building model 0\n",
      "MoleculeModel(\n",
      "  (sigmoid): Sigmoid()\n",
      "  (encoder): MPN(\n",
      "    (encoder): ModuleList(\n",
      "      (0): MPNEncoder(\n",
      "        (dropout_layer): Dropout(p=0.0, inplace=False)\n",
      "        (act_func): ReLU()\n",
      "        (W_i): Linear(in_features=147, out_features=600, bias=False)\n",
      "        (W_h): Linear(in_features=600, out_features=600, bias=False)\n",
      "        (W_o): Linear(in_features=733, out_features=600, bias=True)\n",
      "      )\n",
      "    )\n",
      "  )\n",
      "  (ffn): Sequential(\n",
      "    (0): Dropout(p=0.0, inplace=False)\n",
      "    (1): Linear(in_features=800, out_features=600, bias=True)\n",
      "    (2): ReLU()\n",
      "    (3): Dropout(p=0.0, inplace=False)\n",
      "    (4): Linear(in_features=600, out_features=600, bias=True)\n",
      "    (5): ReLU()\n",
      "    (6): Dropout(p=0.0, inplace=False)\n",
      "    (7): Linear(in_features=600, out_features=1, bias=True)\n",
      "  )\n",
      ")\n",
      "Number of parameters = 1,731,001\n",
      "  0%|          | 0/30 [00:00<?, ?it/s]Epoch 0\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:12,  2.96it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:11,  3.19it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:10,  3.36it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.51it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.60it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.68it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.67it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:08,  3.63it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.80it/s]\u001b[ALoss = 2.7293e-01, PNorm = 52.7591, GNorm = 0.5573, lr_0 = 2.3378e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:08,  3.39it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:03<00:07,  3.53it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.68it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.59it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.69it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:06,  3.61it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  3.70it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  3.75it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.84it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:04,  3.96it/s]\u001b[ALoss = 1.3863e-01, PNorm = 52.7769, GNorm = 0.0390, lr_0 = 3.5541e-04\n",
      "\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  4.02it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.94it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.06it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.37it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:04,  3.36it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.47it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.75it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:07<00:02,  3.80it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.82it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  3.99it/s]\u001b[ALoss = 1.6213e-01, PNorm = 52.7851, GNorm = 0.1459, lr_0 = 4.7703e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.74it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:01,  3.58it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  3.65it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.63it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.71it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.23it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  3.35it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  3.41it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  3.90it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  7.86it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00,  8.26it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  7.89it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00,  8.22it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.415584\n",
      "  3%|▎         | 1/30 [00:10<05:17, 10.95s/it]Epoch 1\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.29it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:07,  4.51it/s]\u001b[ALoss = 6.5917e-02, PNorm = 52.8219, GNorm = 0.0963, lr_0 = 6.1081e-04\n",
      "\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.66it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.70it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.64it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:06,  4.79it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.63it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.68it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:01<00:06,  4.80it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.39it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.49it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 12/38 [00:02<00:05,  4.66it/s]\u001b[ALoss = 1.1197e-01, PNorm = 52.8375, GNorm = 0.1479, lr_0 = 7.3243e-04\n",
      "\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.58it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.59it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:04,  4.71it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:04,  4.96it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.91it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:03<00:04,  4.48it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.36it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.43it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:04,  4.24it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.38it/s]\u001b[ALoss = 1.2145e-01, PNorm = 52.8839, GNorm = 0.6195, lr_0 = 8.5405e-04\n",
      "\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.29it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.18it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:03,  4.10it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.15it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:05<00:02,  4.38it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.15it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.24it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.18it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.17it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.22it/s]\u001b[ALoss = 5.8090e-02, PNorm = 52.9504, GNorm = 0.1176, lr_0 = 9.7568e-04\n",
      "\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.20it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.19it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.33it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.27it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.40it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.00it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 13.94it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.35it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.744589\n",
      "  7%|▋         | 2/30 [00:19<04:49, 10.34s/it]Epoch 2\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.89it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.15it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.19it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.25it/s]\u001b[ALoss = 1.2168e-01, PNorm = 52.9895, GNorm = 0.1873, lr_0 = 9.8238e-04\n",
      "\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.35it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.48it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.58it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.61it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:01<00:06,  4.52it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.64it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.35it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.28it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:06,  4.16it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.12it/s]\u001b[ALoss = 9.2229e-02, PNorm = 53.0779, GNorm = 0.3746, lr_0 = 9.6078e-04\n",
      "\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.95it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.03it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:05,  3.98it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.07it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.20it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.20it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:04,  4.05it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.33it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.44it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.30it/s]\u001b[ALoss = 6.5127e-02, PNorm = 53.1394, GNorm = 0.1254, lr_0 = 9.3966e-04\n",
      "\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.35it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:02,  4.28it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.36it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.41it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.11it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.26it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.88it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  3.89it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.06it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.30it/s]\u001b[ALoss = 7.7366e-02, PNorm = 53.1823, GNorm = 0.1842, lr_0 = 9.1901e-04\n",
      "\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.09it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.12it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.18it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  4.62it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.90it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.52it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.784632\n",
      " 10%|█         | 3/30 [00:29<04:31, 10.04s/it]Epoch 3\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  4.73it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.25it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.39it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.43it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.60it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.32it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.38it/s]\u001b[ALoss = 8.2440e-02, PNorm = 53.2561, GNorm = 0.4148, lr_0 = 8.9681e-04\n",
      "\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.45it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.23it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.23it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.08it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.25it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  4.09it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.11it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.98it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  3.87it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:06,  3.25it/s]\u001b[ALoss = 8.4577e-02, PNorm = 53.3167, GNorm = 0.1641, lr_0 = 8.7710e-04\n",
      "\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.48it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:05,  3.70it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  3.70it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.78it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:04,  3.84it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.01it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.99it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.02it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  4.00it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.89it/s]\u001b[ALoss = 6.7267e-02, PNorm = 53.3820, GNorm = 0.1814, lr_0 = 8.5782e-04\n",
      "\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.95it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.06it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:02,  3.96it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.14it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.07it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.87it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.92it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.03it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.13it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.16it/s]\u001b[ALoss = 6.9376e-02, PNorm = 53.4482, GNorm = 0.2627, lr_0 = 8.3897e-04\n",
      "\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.83it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.71it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.31it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.792208\n",
      " 13%|█▎        | 4/30 [00:38<04:19,  9.97s/it]Epoch 4\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:10,  3.67it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:09,  3.72it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:09,  3.77it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:08,  3.87it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:08,  3.87it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.75it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.87it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:07,  4.04it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  4.08it/s]\u001b[ALoss = 5.9091e-02, PNorm = 53.5296, GNorm = 0.2265, lr_0 = 8.2053e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.92it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  3.94it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  3.93it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  4.09it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.08it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.12it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.34it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.18it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.18it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.87it/s]\u001b[ALoss = 8.1418e-02, PNorm = 53.6079, GNorm = 0.1168, lr_0 = 8.0249e-04\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 20/38 [00:05<00:04,  3.95it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  4.19it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:04,  3.90it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  3.84it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  4.01it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.87it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.99it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.95it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.84it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.03it/s]\u001b[ALoss = 4.9421e-02, PNorm = 53.6631, GNorm = 0.1953, lr_0 = 7.8485e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.25it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.37it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.47it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  4.17it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.96it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  3.99it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.28it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.32it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.88it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 13.04it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.41it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.781385\n",
      " 17%|█▋        | 5/30 [00:48<04:06,  9.88s/it]Epoch 5\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  4.73it/s]\u001b[ALoss = 6.5729e-02, PNorm = 53.7127, GNorm = 0.1592, lr_0 = 7.6760e-04\n",
      "\n",
      "  5%|▌         | 2/38 [00:00<00:07,  4.58it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.61it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.61it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.60it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.53it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.68it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.45it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.36it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.30it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.18it/s]\u001b[ALoss = 4.6857e-02, PNorm = 53.7900, GNorm = 0.0462, lr_0 = 7.5073e-04\n",
      "\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.18it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:06,  4.12it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.19it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.24it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.35it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:05,  4.10it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.17it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.14it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.03it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:04,  4.07it/s]\u001b[ALoss = 6.8831e-02, PNorm = 53.8408, GNorm = 0.1788, lr_0 = 7.3423e-04\n",
      "\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.09it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.00it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.05it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:03,  4.00it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.81it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.78it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  3.66it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  3.62it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:02,  3.68it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.76it/s]\u001b[ALoss = 7.7740e-02, PNorm = 53.8839, GNorm = 0.1632, lr_0 = 7.1809e-04\n",
      "\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  3.98it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.99it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.95it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.12it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.00it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  3.80it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.40it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.66it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00, 11.67it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 12.46it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.821429\n",
      " 20%|██        | 6/30 [00:58<03:55,  9.82s/it]Epoch 6\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.92it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.10it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.02it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:08,  3.98it/s]\u001b[ALoss = 1.0248e-02, PNorm = 53.9787, GNorm = 0.0269, lr_0 = 7.0074e-04\n",
      "\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.16it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.13it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.05it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:07,  4.06it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.87it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.83it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.77it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  3.90it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  4.01it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.20it/s]\u001b[ALoss = 7.9444e-02, PNorm = 53.9929, GNorm = 0.0996, lr_0 = 6.8534e-04\n",
      "\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.91it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  3.73it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  3.91it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.93it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.03it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  4.02it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.96it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.10it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.05it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.11it/s]\u001b[ALoss = 3.7872e-02, PNorm = 54.0517, GNorm = 0.1683, lr_0 = 6.7028e-04\n",
      "\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.10it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.96it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.87it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.88it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.16it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.05it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.02it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.12it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  4.15it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:00,  4.36it/s]\u001b[ALoss = 8.5063e-02, PNorm = 54.1025, GNorm = 0.1936, lr_0 = 6.5554e-04\n",
      "\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.33it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.29it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  3.82it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.36it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.53it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.21it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.808442\n",
      " 23%|██▎       | 7/30 [01:08<03:45,  9.80s/it]Epoch 7\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.49it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.36it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.30it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.37it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.22it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.09it/s]\u001b[ALoss = 3.0229e-02, PNorm = 54.1861, GNorm = 0.0473, lr_0 = 6.4114e-04\n",
      "\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.00it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:07,  3.94it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.93it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.12it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.04it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.00it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.96it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.78it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:06,  3.82it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.04it/s]\u001b[ALoss = 6.9525e-02, PNorm = 54.2064, GNorm = 0.2042, lr_0 = 6.2704e-04\n",
      "\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.12it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.00it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.98it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.18it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  4.18it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.36it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.22it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.03it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.08it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:02,  4.15it/s]\u001b[ALoss = 5.1295e-02, PNorm = 54.2750, GNorm = 0.2952, lr_0 = 6.1326e-04\n",
      "\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.13it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.32it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.10it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.11it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.12it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  3.74it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.80it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:00,  4.07it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.15it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.02it/s]\u001b[ALoss = 5.6822e-02, PNorm = 54.2984, GNorm = 0.1236, lr_0 = 5.9978e-04\n",
      "\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  3.85it/s]\u001b[ALoss = 1.2951e-01, PNorm = 54.2996, GNorm = 0.5251, lr_0 = 5.9845e-04\n",
      "\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.39it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.22it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 11.96it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.819264\n",
      " 27%|██▋       | 8/30 [01:17<03:35,  9.77s/it]Epoch 8\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:10,  3.51it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:09,  3.69it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.00it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:08,  3.97it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:08,  3.90it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.99it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  3.88it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:07,  4.09it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.16it/s]\u001b[ALoss = 5.1058e-02, PNorm = 54.3473, GNorm = 0.2207, lr_0 = 5.8529e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.08it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.27it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.19it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:05,  4.42it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.23it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.20it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.25it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.16it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.20it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.17it/s]\u001b[ALoss = 4.1749e-02, PNorm = 54.4010, GNorm = 0.1657, lr_0 = 5.7243e-04\n",
      "\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.23it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  4.18it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:04,  3.90it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:04,  3.65it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  3.70it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.84it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.89it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.85it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  3.70it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  3.71it/s]\u001b[ALoss = 4.4345e-02, PNorm = 54.4440, GNorm = 0.2669, lr_0 = 5.5985e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:07<00:02,  3.84it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.91it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  3.72it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.60it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.39it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  3.56it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  3.63it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  3.65it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  3.92it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 11.65it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00, 10.96it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 11.75it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.808442\n",
      " 30%|███       | 9/30 [01:27<03:27,  9.86s/it]Epoch 9\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.86it/s]\u001b[ALoss = 3.4560e-02, PNorm = 54.4630, GNorm = 0.1052, lr_0 = 5.4754e-04\n",
      "\n",
      "  5%|▌         | 2/38 [00:00<00:09,  3.81it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:09,  3.79it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:08,  4.00it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:08,  4.05it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.82it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.81it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:07,  3.81it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.86it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.96it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  3.86it/s]\u001b[ALoss = 3.8859e-02, PNorm = 54.4995, GNorm = 0.0293, lr_0 = 5.3551e-04\n",
      "\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  3.72it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.58it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.51it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:06,  3.63it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.60it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  3.59it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.61it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:06,  2.86it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:05,  3.05it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:05,  3.19it/s]\u001b[ALoss = 4.3607e-02, PNorm = 54.5342, GNorm = 0.2398, lr_0 = 5.2374e-04\n",
      "\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.34it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.54it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.53it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.61it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.70it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:07<00:02,  3.71it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.83it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.70it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.69it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:01,  3.82it/s]\u001b[ALoss = 4.5976e-02, PNorm = 54.5755, GNorm = 0.1182, lr_0 = 5.1222e-04\n",
      "\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  3.91it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.97it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.77it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.85it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  3.80it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  3.53it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  4.06it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  9.48it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.65it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 10.48it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.798701\n",
      " 33%|███▎      | 10/30 [01:38<03:22, 10.14s/it]Epoch 10\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:10,  3.55it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:10,  3.56it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:09,  3.54it/s]\u001b[ALoss = 4.5689e-02, PNorm = 54.5980, GNorm = 0.3064, lr_0 = 5.0096e-04\n",
      "\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.57it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.47it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:09,  3.40it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:02<00:08,  3.45it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:08,  3.58it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:08,  3.56it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.57it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:03<00:07,  3.62it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.68it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:07,  3.43it/s]\u001b[ALoss = 3.0576e-02, PNorm = 54.6251, GNorm = 0.0875, lr_0 = 4.8995e-04\n",
      "\n",
      " 37%|███▋      | 14/38 [00:04<00:07,  3.27it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:07,  3.28it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.32it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:06,  3.22it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:05<00:05,  3.34it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.41it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:05,  3.50it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:06<00:04,  3.64it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.75it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:03,  3.76it/s]\u001b[ALoss = 3.9454e-02, PNorm = 54.6620, GNorm = 0.2505, lr_0 = 4.7918e-04\n",
      "\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.67it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:07<00:03,  3.45it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.47it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:07<00:03,  3.59it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:08<00:02,  3.45it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.63it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.25it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:02,  3.32it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.56it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.70it/s]\u001b[ALoss = 5.2270e-02, PNorm = 54.6862, GNorm = 0.1145, lr_0 = 4.6865e-04\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.74it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.58it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.62it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  3.68it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  4.09it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 11.11it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00, 10.28it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 11.00it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.819264\n",
      " 37%|███▋      | 11/30 [01:49<03:18, 10.44s/it]Epoch 11\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:12,  2.86it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:11,  3.13it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:10,  3.34it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.43it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.58it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:09,  3.50it/s]\u001b[ALoss = 4.0428e-02, PNorm = 54.7296, GNorm = 0.1281, lr_0 = 4.5733e-04\n",
      "\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.58it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:08,  3.48it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:08,  3.60it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:08,  3.47it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:03<00:07,  3.39it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.53it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:07,  3.52it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.60it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:06,  3.47it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.51it/s]\u001b[ALoss = 2.9789e-02, PNorm = 54.7592, GNorm = 0.2076, lr_0 = 4.4728e-04\n",
      "\n",
      " 45%|████▍     | 17/38 [00:04<00:06,  3.38it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:05<00:05,  3.45it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.34it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:05,  3.60it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.51it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.57it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.60it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.68it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:07<00:03,  3.55it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.39it/s]\u001b[ALoss = 2.3497e-02, PNorm = 54.7973, GNorm = 0.2139, lr_0 = 4.3745e-04\n",
      "\n",
      " 71%|███████   | 27/38 [00:07<00:03,  3.45it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.57it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.40it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.43it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:02,  3.47it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.51it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.40it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.43it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:10<00:00,  3.36it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.41it/s]\u001b[ALoss = 4.8135e-02, PNorm = 54.8070, GNorm = 0.5160, lr_0 = 4.2783e-04\n",
      "\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  3.46it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  3.92it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 10.74it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.81it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 10.59it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.800866\n",
      " 40%|████      | 12/30 [02:01<03:12, 10.67s/it]Epoch 12\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:12,  2.91it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:11,  3.14it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:11,  3.12it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:10,  3.30it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:10,  3.25it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:10,  3.15it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:02<00:11,  2.67it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:12,  2.36it/s]\u001b[ALoss = 3.3050e-02, PNorm = 54.8362, GNorm = 0.2486, lr_0 = 4.1843e-04\n",
      "\n",
      " 24%|██▎       | 9/38 [00:03<00:11,  2.42it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:03<00:13,  2.05it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:04<00:13,  1.99it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:04<00:12,  2.17it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:05<00:11,  2.19it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:05<00:10,  2.20it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:06<00:10,  2.14it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:06<00:10,  2.20it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:07<00:09,  2.25it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:07<00:07,  2.55it/s]\u001b[ALoss = 2.1585e-02, PNorm = 54.8696, GNorm = 0.0132, lr_0 = 4.0923e-04\n",
      "\n",
      " 50%|█████     | 19/38 [00:07<00:07,  2.70it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:07<00:06,  2.89it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:08<00:05,  2.92it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:08<00:05,  3.14it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:08<00:04,  3.14it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:09<00:04,  3.32it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:09<00:03,  3.41it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:09<00:03,  3.40it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:10<00:03,  3.30it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:10<00:03,  3.31it/s]\u001b[ALoss = 3.8341e-02, PNorm = 54.8758, GNorm = 0.2038, lr_0 = 4.0024e-04\n",
      "\n",
      " 76%|███████▋  | 29/38 [00:10<00:02,  3.07it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:11<00:02,  3.09it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:11<00:02,  3.21it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:11<00:02,  2.98it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:12<00:01,  3.06it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:12<00:01,  3.12it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:12<00:00,  3.27it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:12<00:00,  3.12it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:13<00:00,  3.15it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:13<00:00,  3.69it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  9.66it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.48it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00,  9.54it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.832251\n",
      " 43%|████▎     | 13/30 [02:15<03:18, 11.67s/it]Epoch 13\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[ALoss = 2.5121e-02, PNorm = 54.9104, GNorm = 0.2968, lr_0 = 3.9144e-04\n",
      "\n",
      "  3%|▎         | 1/38 [00:00<00:13,  2.74it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:12,  2.91it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:11,  3.05it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:10,  3.10it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:10,  3.11it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:10,  3.02it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:02<00:10,  3.09it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:09,  3.10it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:08,  3.23it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:03<00:08,  3.22it/s]\u001b[ALoss = 2.9696e-02, PNorm = 54.9369, GNorm = 0.2053, lr_0 = 3.8283e-04\n",
      "\n",
      " 29%|██▉       | 11/38 [00:03<00:08,  3.29it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.38it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:07,  3.54it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:04<00:06,  3.60it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:06,  3.57it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.39it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:05<00:06,  3.31it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:05<00:06,  3.32it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.24it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:06<00:05,  3.26it/s]\u001b[ALoss = 1.9012e-02, PNorm = 54.9672, GNorm = 0.1928, lr_0 = 3.7442e-04\n",
      "\n",
      " 55%|█████▌    | 21/38 [00:06<00:04,  3.42it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.58it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.68it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:07<00:03,  3.52it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:07<00:03,  3.38it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.40it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:08<00:03,  3.34it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:08<00:02,  3.45it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.42it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.39it/s]\u001b[ALoss = 5.0953e-02, PNorm = 54.9773, GNorm = 0.4282, lr_0 = 3.6619e-04\n",
      "\n",
      " 82%|████████▏ | 31/38 [00:09<00:02,  3.39it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.31it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.49it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:10<00:01,  3.52it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:10<00:00,  3.68it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.48it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:11<00:00,  3.19it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:11<00:00,  3.64it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  9.33it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.40it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 10.19it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.812771\n",
      " 47%|████▋     | 14/30 [02:26<03:06, 11.68s/it]Epoch 14\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:11,  3.14it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:11,  3.14it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:11,  3.13it/s]\u001b[ALoss = 3.6156e-02, PNorm = 55.0029, GNorm = 0.3406, lr_0 = 3.5735e-04\n",
      "\n",
      " 11%|█         | 4/38 [00:01<00:11,  3.05it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:10,  3.07it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:10,  3.14it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:02<00:09,  3.23it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:09,  3.25it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:08,  3.47it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:03<00:07,  3.61it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:03<00:07,  3.48it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.39it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.59it/s]\u001b[ALoss = 2.1339e-02, PNorm = 55.0327, GNorm = 1.1690, lr_0 = 3.4949e-04\n",
      "\n",
      " 37%|███▋      | 14/38 [00:04<00:06,  3.61it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:06,  3.60it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.53it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:05<00:06,  3.38it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:05<00:06,  3.28it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.39it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:05,  3.38it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:06<00:05,  3.10it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:05,  3.03it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.19it/s]\u001b[ALoss = 2.5532e-02, PNorm = 55.0440, GNorm = 0.1150, lr_0 = 3.4181e-04\n",
      "\n",
      " 63%|██████▎   | 24/38 [00:07<00:04,  3.07it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:07<00:04,  3.22it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.30it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:08<00:03,  3.49it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:08<00:02,  3.45it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.27it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:09<00:02,  3.13it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:09<00:02,  3.14it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.11it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:10<00:01,  3.15it/s]\u001b[ALoss = 3.6495e-02, PNorm = 55.0676, GNorm = 0.3096, lr_0 = 3.3430e-04\n",
      "\n",
      " 89%|████████▉ | 34/38 [00:10<00:01,  3.35it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:10<00:00,  3.37it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.43it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:11<00:00,  3.51it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:11<00:00,  4.00it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  9.41it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.52it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00,  5.93it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.819264\n",
      " 50%|█████     | 15/30 [02:38<02:56, 11.79s/it]Epoch 15\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:12,  3.05it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:10,  3.28it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:10,  3.30it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.42it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.58it/s]\u001b[ALoss = 3.4307e-02, PNorm = 55.0855, GNorm = 0.2187, lr_0 = 3.2695e-04\n",
      "\n",
      " 16%|█▌        | 6/38 [00:01<00:09,  3.55it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.58it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:08,  3.55it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.64it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.82it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  3.99it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  4.01it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.77it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.63it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:06,  3.51it/s]\u001b[ALoss = 3.2126e-02, PNorm = 55.1069, GNorm = 0.5945, lr_0 = 3.1976e-04\n",
      "\n",
      " 42%|████▏     | 16/38 [00:04<00:06,  3.40it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  3.54it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.44it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.37it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:05,  3.59it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.65it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.37it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.43it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.54it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.47it/s]\u001b[ALoss = 1.6870e-02, PNorm = 55.1168, GNorm = 0.0184, lr_0 = 3.1273e-04\n",
      "\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.46it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:07<00:03,  3.44it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.54it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.33it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.36it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:02,  3.28it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.16it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.32it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.40it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.43it/s]\u001b[ALoss = 2.9520e-02, PNorm = 55.1257, GNorm = 0.2275, lr_0 = 3.0586e-04\n",
      "\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.46it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  3.16it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  3.66it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 20%|██        | 1/5 [00:00<00:00,  9.63it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00,  9.97it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 11.00it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.797619\n",
      " 53%|█████▎    | 16/30 [02:50<02:42, 11.62s/it]Epoch 16\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:10,  3.68it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:09,  3.77it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:09,  3.86it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:08,  3.92it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.65it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.57it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.68it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:08,  3.36it/s]\u001b[ALoss = 2.3799e-02, PNorm = 55.1452, GNorm = 0.2996, lr_0 = 2.9847e-04\n",
      "\n",
      " 24%|██▎       | 9/38 [00:02<00:09,  3.10it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:08,  3.14it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:03<00:08,  3.23it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.42it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:07,  3.55it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:04<00:08,  2.96it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:04<00:07,  2.89it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:07,  2.95it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:05<00:06,  3.02it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:05<00:06,  3.25it/s]\u001b[ALoss = 3.0081e-02, PNorm = 55.1613, GNorm = 0.4542, lr_0 = 2.9191e-04\n",
      "\n",
      " 50%|█████     | 19/38 [00:05<00:05,  3.21it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:06<00:05,  3.45it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:06<00:04,  3.61it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:06<00:04,  3.51it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:06<00:04,  3.52it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:07<00:03,  3.67it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:07<00:03,  3.67it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:07<00:03,  3.84it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:07<00:02,  3.98it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:08<00:02,  3.67it/s]\u001b[ALoss = 3.2605e-02, PNorm = 55.1782, GNorm = 0.1895, lr_0 = 2.8549e-04\n",
      "\n",
      " 76%|███████▋  | 29/38 [00:08<00:02,  3.87it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:08<00:02,  3.85it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:08<00:01,  4.06it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:09<00:01,  3.96it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:09<00:01,  3.71it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:09<00:01,  3.82it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.97it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:10<00:00,  3.92it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:10<00:00,  4.10it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:10<00:00,  4.79it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 11.94it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00, 11.16it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 11.99it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.825758\n",
      " 57%|█████▋    | 17/30 [03:00<02:28, 11.41s/it]Epoch 17\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[ALoss = 2.4652e-03, PNorm = 55.2038, GNorm = 0.0360, lr_0 = 2.7922e-04\n",
      "\n",
      "  3%|▎         | 1/38 [00:00<00:09,  4.06it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:09,  4.00it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.05it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:08,  3.84it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:08,  3.91it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 6/38 [00:01<00:08,  3.91it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  3.96it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:07,  3.82it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.92it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.88it/s]\u001b[ALoss = 3.3812e-02, PNorm = 55.2093, GNorm = 0.1212, lr_0 = 2.7308e-04\n",
      "\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.60it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.61it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.62it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.62it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.90it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  4.08it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.01it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.02it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.95it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  4.01it/s]\u001b[ALoss = 1.9849e-02, PNorm = 55.2268, GNorm = 0.1803, lr_0 = 2.6708e-04\n",
      "\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  4.06it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.29it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.15it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.99it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.04it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.92it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.98it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.89it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.00it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:02,  3.79it/s]\u001b[ALoss = 3.0470e-02, PNorm = 55.2382, GNorm = 0.1995, lr_0 = 2.6121e-04\n",
      "\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.79it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  3.85it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.92it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:00,  4.13it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.30it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  4.35it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.33it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.93it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.56it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.25it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.820346\n",
      " 60%|██████    | 18/30 [03:10<02:11, 10.94s/it]Epoch 18\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.84it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.06it/s]\u001b[ALoss = 2.1653e-02, PNorm = 55.2496, GNorm = 0.2150, lr_0 = 2.5547e-04\n",
      "\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.07it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:08,  4.01it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.32it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.46it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.50it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.34it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.49it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.25it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.20it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.27it/s]\u001b[ALoss = 2.4804e-02, PNorm = 55.2675, GNorm = 0.0463, lr_0 = 2.4985e-04\n",
      "\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  4.12it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.05it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.30it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.26it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.35it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.19it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.15it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.34it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.40it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.58it/s]\u001b[ALoss = 1.6780e-02, PNorm = 55.2826, GNorm = 0.1632, lr_0 = 2.4436e-04\n",
      "\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.67it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:02,  4.82it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:03,  4.33it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.32it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.40it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.34it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.41it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.17it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.21it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.37it/s]\u001b[ALoss = 2.5067e-02, PNorm = 55.2985, GNorm = 0.3720, lr_0 = 2.3899e-04\n",
      "\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.38it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.73it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.73it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.70it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.61it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.27it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 13.93it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.57it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.826840\n",
      " 63%|██████▎   | 19/30 [03:19<01:53, 10.33s/it]Epoch 19\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  4.78it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:07,  4.73it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.79it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.51it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.53it/s]\u001b[ALoss = 6.9896e-03, PNorm = 55.3120, GNorm = 0.1909, lr_0 = 2.3322e-04\n",
      "\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.13it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.19it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.45it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:01<00:06,  4.72it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:05,  4.77it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:05,  4.85it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:05,  4.44it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.67it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.31it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.27it/s]\u001b[ALoss = 2.6251e-02, PNorm = 55.3183, GNorm = 0.2605, lr_0 = 2.2809e-04\n",
      "\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.20it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.38it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.55it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.67it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.72it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.66it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.52it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.43it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.28it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.56it/s]\u001b[ALoss = 1.9517e-02, PNorm = 55.3338, GNorm = 0.1347, lr_0 = 2.2308e-04\n",
      "\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.62it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:05<00:02,  4.60it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.73it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.46it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.47it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.60it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.61it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.42it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.33it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.34it/s]\u001b[ALoss = 2.5235e-02, PNorm = 55.3458, GNorm = 0.9682, lr_0 = 2.1817e-04\n",
      "\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.34it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.32it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  4.94it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.08it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.73it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.821429\n",
      " 67%|██████▋   | 20/30 [03:28<01:38,  9.86s/it]Epoch 20\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.81it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:10,  3.34it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:10,  3.33it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.60it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:09,  3.51it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:09,  3.34it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:08,  3.83it/s]\u001b[ALoss = 1.5396e-02, PNorm = 55.3554, GNorm = 0.3796, lr_0 = 2.1338e-04\n",
      "\n",
      " 21%|██        | 8/38 [00:02<00:07,  3.99it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  4.09it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.30it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.40it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:05,  4.67it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:05,  4.68it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.47it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 15/38 [00:03<00:05,  4.55it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:04,  4.58it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:04,  4.58it/s]\u001b[ALoss = 1.5665e-02, PNorm = 55.3720, GNorm = 0.4936, lr_0 = 2.0869e-04\n",
      "\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.54it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.29it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.49it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:03,  4.53it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.64it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.64it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:02,  4.84it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.60it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.91it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.01it/s]\u001b[ALoss = 2.3602e-02, PNorm = 55.3828, GNorm = 0.1519, lr_0 = 2.0410e-04\n",
      "\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.18it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.28it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.47it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.63it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.62it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.78it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:00,  4.67it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.73it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.58it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.43it/s]\u001b[ALoss = 1.8169e-02, PNorm = 55.4020, GNorm = 0.2707, lr_0 = 1.9961e-04\n",
      "\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.16it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.12it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.82it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.825758\n",
      " 70%|███████   | 21/30 [03:37<01:26,  9.65s/it]Epoch 21\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:09,  3.98it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.27it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.41it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.29it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.56it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:06,  4.59it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.49it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.55it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:01<00:06,  4.57it/s]\u001b[ALoss = 2.6762e-02, PNorm = 55.4061, GNorm = 0.1163, lr_0 = 1.9523e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:05,  4.67it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.46it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:05,  4.71it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.81it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:04,  4.81it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.57it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:04,  4.59it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.75it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:03<00:04,  4.56it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.58it/s]\u001b[ALoss = 2.4187e-02, PNorm = 55.4137, GNorm = 0.1436, lr_0 = 1.9094e-04\n",
      "\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.61it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.73it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.69it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:04<00:03,  4.76it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.52it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.67it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.46it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:05<00:02,  4.41it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.47it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:01,  4.59it/s]\u001b[ALoss = 1.3876e-02, PNorm = 55.4276, GNorm = 0.4136, lr_0 = 1.8674e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.70it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.63it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:06<00:01,  4.65it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.70it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.44it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.42it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:07<00:00,  4.34it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.35it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.09it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.66it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 14.33it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.819264\n",
      " 73%|███████▎  | 22/30 [03:46<01:14,  9.32s/it]Epoch 22\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  5.23it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:07,  5.04it/s]\u001b[ALoss = 3.7887e-02, PNorm = 55.4375, GNorm = 0.1633, lr_0 = 1.8223e-04\n",
      "\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.84it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:06,  5.02it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.61it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.37it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.45it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.35it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.31it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.38it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.45it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:05,  4.38it/s]\u001b[ALoss = 8.1324e-03, PNorm = 55.4440, GNorm = 0.2550, lr_0 = 1.7822e-04\n",
      "\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.18it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.41it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.34it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.31it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.59it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.46it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.58it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.66it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.82it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.69it/s]\u001b[ALoss = 1.5367e-02, PNorm = 55.4565, GNorm = 0.0533, lr_0 = 1.7431e-04\n",
      "\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.63it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.47it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.57it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.67it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:05<00:02,  4.82it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.54it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.44it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.41it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.53it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.58it/s]\u001b[ALoss = 1.9060e-02, PNorm = 55.4644, GNorm = 0.0087, lr_0 = 1.7047e-04\n",
      "\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.87it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.98it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.82it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:07<00:00,  4.71it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.61it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.34it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 13.93it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.58it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.825758\n",
      " 77%|███████▋  | 23/30 [03:54<01:03,  9.10s/it]Epoch 23\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  4.88it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.38it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.14it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:08,  4.10it/s]\u001b[ALoss = 1.7600e-02, PNorm = 55.4669, GNorm = 0.0449, lr_0 = 1.6673e-04\n",
      "\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.20it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.31it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.61it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.69it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.82it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:05,  4.83it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.49it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:05,  4.52it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.58it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.54it/s]\u001b[ALoss = 1.8010e-02, PNorm = 55.4818, GNorm = 0.4858, lr_0 = 1.6306e-04\n",
      "\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.18it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:04,  4.43it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.44it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.69it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.62it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.55it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.41it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.48it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.67it/s]\u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 24/38 [00:05<00:02,  4.77it/s]\u001b[ALoss = 2.1765e-02, PNorm = 55.4920, GNorm = 0.4932, lr_0 = 1.5948e-04\n",
      "\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.58it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.60it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.53it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.70it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:01,  4.85it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.59it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.63it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.87it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.79it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.59it/s]\u001b[ALoss = 9.0661e-03, PNorm = 55.5048, GNorm = 0.3372, lr_0 = 1.5597e-04\n",
      "\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.51it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:07<00:00,  4.68it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.67it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.14it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.62it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 14.14it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.831169\n",
      " 80%|████████  | 24/30 [04:03<00:53,  8.96s/it]Epoch 24\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:07,  4.95it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:07,  4.75it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.65it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.66it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:06,  4.81it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:06,  4.94it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:06,  4.79it/s]\u001b[ALoss = 1.1037e-02, PNorm = 55.5127, GNorm = 0.0889, lr_0 = 1.5221e-04\n",
      "\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.62it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:01<00:06,  4.63it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:05,  4.76it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:05,  4.59it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:06,  4.30it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.33it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.52it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:04,  4.65it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:04,  4.67it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.64it/s]\u001b[ALoss = 1.2082e-02, PNorm = 55.5190, GNorm = 0.0535, lr_0 = 1.4886e-04\n",
      "\n",
      " 47%|████▋     | 18/38 [00:03<00:04,  4.55it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.65it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.55it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.58it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.86it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:04<00:03,  4.60it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.64it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.49it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.56it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:05<00:02,  4.68it/s]\u001b[ALoss = 1.8918e-02, PNorm = 55.5242, GNorm = 0.5904, lr_0 = 1.4559e-04\n",
      "\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.42it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.47it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.69it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.67it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:06<00:01,  4.59it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.57it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.52it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.63it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:07<00:00,  4.69it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.59it/s]\u001b[ALoss = 1.4893e-02, PNorm = 55.5265, GNorm = 0.1825, lr_0 = 1.4239e-04\n",
      "\n",
      "100%|██████████| 38/38 [00:08<00:00,  5.11it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.03it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.58it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.829004\n",
      " 83%|████████▎ | 25/30 [04:11<00:44,  8.83s/it]Epoch 25\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.52it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.45it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:07,  4.53it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.61it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.52it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.33it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.21it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:06,  4.29it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:06,  4.48it/s]\u001b[ALoss = 1.7521e-02, PNorm = 55.5381, GNorm = 0.4339, lr_0 = 1.3926e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:06,  4.45it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:06,  4.31it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:02<00:05,  4.35it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:02<00:05,  4.39it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.52it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:04,  4.67it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.36it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:03<00:04,  4.57it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.78it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.64it/s]\u001b[ALoss = 1.6681e-02, PNorm = 55.5510, GNorm = 0.1514, lr_0 = 1.3620e-04\n",
      "\n",
      " 53%|█████▎    | 20/38 [00:04<00:03,  4.77it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:04<00:03,  4.73it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:04<00:03,  4.65it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.40it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.50it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:05<00:02,  4.56it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:05<00:02,  4.40it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.35it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.34it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.42it/s]\u001b[ALoss = 7.1846e-03, PNorm = 55.5600, GNorm = 0.0383, lr_0 = 1.3320e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:06<00:01,  4.43it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:06<00:01,  4.46it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.41it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:07<00:01,  4.35it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:07<00:00,  4.25it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:07<00:00,  4.22it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.19it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:08<00:00,  4.30it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:08<00:00,  4.91it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 14.27it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 13.61it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.831169\n",
      " 87%|████████▋ | 26/30 [04:20<00:35,  8.83s/it]Epoch 26\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.47it/s]\u001b[ALoss = 1.6307e-02, PNorm = 55.5650, GNorm = 0.0126, lr_0 = 1.3028e-04\n",
      "\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.21it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.21it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:08,  4.19it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.20it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.21it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  3.93it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:07,  3.89it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  4.13it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.56it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.52it/s]\u001b[ALoss = 1.2967e-02, PNorm = 55.5669, GNorm = 0.5497, lr_0 = 1.2741e-04\n",
      "\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.59it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.99it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:05,  4.02it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.23it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:03<00:05,  4.40it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:04,  4.53it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.58it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  4.30it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:04<00:04,  4.33it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:03,  4.29it/s]\u001b[ALoss = 6.8977e-03, PNorm = 55.5698, GNorm = 0.3511, lr_0 = 1.2461e-04\n",
      "\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.37it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.58it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:05<00:03,  4.42it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.26it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:02,  4.23it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.12it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.18it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:06<00:02,  4.31it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.21it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.99it/s]\u001b[ALoss = 9.1415e-03, PNorm = 55.5790, GNorm = 0.0086, lr_0 = 1.2187e-04\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.15it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.48it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.54it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  3.77it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  3.87it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.11it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.82it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.74it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.31it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.831169\n",
      " 90%|█████████ | 27/30 [04:30<00:27,  9.06s/it]Epoch 27\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.44it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.37it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.15it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:07,  4.32it/s]\u001b[ALoss = 1.7404e-02, PNorm = 55.5857, GNorm = 0.0210, lr_0 = 1.1893e-04\n",
      "\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.28it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:07,  4.20it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.06it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:07,  4.05it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.84it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.73it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.79it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  3.95it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.79it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.85it/s]\u001b[ALoss = 9.3028e-03, PNorm = 55.5934, GNorm = 0.2849, lr_0 = 1.1632e-04\n",
      "\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  4.00it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  3.97it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.11it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.04it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.84it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  3.91it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.93it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:03,  4.09it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.21it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  4.06it/s]\u001b[ALoss = 1.6520e-02, PNorm = 55.6010, GNorm = 0.4081, lr_0 = 1.1376e-04\n",
      "\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.01it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:02,  4.13it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  4.06it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:06<00:02,  4.13it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.12it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.19it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.13it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:07<00:01,  4.08it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.94it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:00,  4.15it/s]\u001b[ALoss = 1.0886e-02, PNorm = 55.6055, GNorm = 0.1042, lr_0 = 1.1126e-04\n",
      "\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  4.26it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:08<00:00,  4.04it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.01it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.53it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.24it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.14it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.831169\n",
      " 93%|█████████▎| 28/30 [04:40<00:18,  9.26s/it]Epoch 28\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:08,  4.57it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:08,  4.36it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:08,  4.17it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:00<00:08,  4.07it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:07,  4.19it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.97it/s]\u001b[ALoss = 8.8876e-03, PNorm = 55.6103, GNorm = 0.0174, lr_0 = 1.0881e-04\n",
      "\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  3.96it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:01<00:07,  4.04it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.71it/s]\u001b[A\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.78it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.80it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:06,  3.77it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.92it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.83it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.94it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  4.12it/s]\u001b[ALoss = 9.2816e-03, PNorm = 55.6164, GNorm = 0.2207, lr_0 = 1.0642e-04\n",
      "\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  4.14it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:04,  4.00it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.99it/s]\u001b[A\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  4.01it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  3.82it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:04,  3.97it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  4.01it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.75it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  3.71it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:03,  3.82it/s]\u001b[ALoss = 1.8869e-02, PNorm = 55.6225, GNorm = 0.1282, lr_0 = 1.0408e-04\n",
      "\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.82it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  3.90it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  3.84it/s]\u001b[A\n",
      " 79%|███████▉  | 30/38 [00:07<00:02,  3.72it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  4.04it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  4.05it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.87it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  4.00it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:08<00:00,  3.88it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  3.80it/s]\u001b[ALoss = 9.0285e-03, PNorm = 55.6243, GNorm = 0.2257, lr_0 = 1.0179e-04\n",
      "\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  3.83it/s]\u001b[ALoss = 3.4690e-02, PNorm = 55.6250, GNorm = 0.6215, lr_0 = 1.0157e-04\n",
      "\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.34it/s]\u001b[A\n",
      "                                               \u001b[A\n",
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 11.96it/s]\u001b[A\n",
      " 60%|██████    | 3/5 [00:00<00:00, 11.20it/s]\u001b[A\n",
      "100%|██████████| 5/5 [00:00<00:00, 12.06it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.830087\n",
      " 97%|█████████▋| 29/30 [04:50<00:09,  9.50s/it]Epoch 29\n",
      "\n",
      "  0%|          | 0/38 [00:00<?, ?it/s]\u001b[A\n",
      "  3%|▎         | 1/38 [00:00<00:11,  3.30it/s]\u001b[A\n",
      "  5%|▌         | 2/38 [00:00<00:10,  3.28it/s]\u001b[A\n",
      "  8%|▊         | 3/38 [00:00<00:10,  3.41it/s]\u001b[A\n",
      " 11%|█         | 4/38 [00:01<00:09,  3.61it/s]\u001b[A\n",
      " 13%|█▎        | 5/38 [00:01<00:08,  3.75it/s]\u001b[A\n",
      " 16%|█▌        | 6/38 [00:01<00:08,  3.92it/s]\u001b[A\n",
      " 18%|█▊        | 7/38 [00:01<00:07,  4.00it/s]\u001b[A\n",
      " 21%|██        | 8/38 [00:02<00:07,  4.06it/s]\u001b[A\n",
      " 24%|██▎       | 9/38 [00:02<00:07,  3.96it/s]\u001b[ALoss = 1.1315e-02, PNorm = 55.6330, GNorm = 0.4958, lr_0 = 1.0000e-04\n",
      "\n",
      " 26%|██▋       | 10/38 [00:02<00:07,  3.88it/s]\u001b[A\n",
      " 29%|██▉       | 11/38 [00:02<00:07,  3.81it/s]\u001b[A\n",
      " 32%|███▏      | 12/38 [00:03<00:07,  3.69it/s]\u001b[A\n",
      " 34%|███▍      | 13/38 [00:03<00:06,  3.68it/s]\u001b[A\n",
      " 37%|███▋      | 14/38 [00:03<00:06,  3.80it/s]\u001b[A\n",
      " 39%|███▉      | 15/38 [00:03<00:05,  3.93it/s]\u001b[A\n",
      " 42%|████▏     | 16/38 [00:04<00:05,  3.74it/s]\u001b[A\n",
      " 45%|████▍     | 17/38 [00:04<00:05,  3.69it/s]\u001b[A\n",
      " 47%|████▋     | 18/38 [00:04<00:05,  3.76it/s]\u001b[A\n",
      " 50%|█████     | 19/38 [00:04<00:04,  3.92it/s]\u001b[ALoss = 6.4568e-03, PNorm = 55.6387, GNorm = 0.0897, lr_0 = 1.0000e-04\n",
      "\n",
      " 53%|█████▎    | 20/38 [00:05<00:04,  4.04it/s]\u001b[A\n",
      " 55%|█████▌    | 21/38 [00:05<00:04,  4.24it/s]\u001b[A\n",
      " 58%|█████▊    | 22/38 [00:05<00:04,  3.90it/s]\u001b[A\n",
      " 61%|██████    | 23/38 [00:05<00:03,  3.83it/s]\u001b[A\n",
      " 63%|██████▎   | 24/38 [00:06<00:03,  3.93it/s]\u001b[A\n",
      " 66%|██████▌   | 25/38 [00:06<00:03,  4.03it/s]\u001b[A\n",
      " 68%|██████▊   | 26/38 [00:06<00:02,  4.05it/s]\u001b[A\n",
      " 71%|███████   | 27/38 [00:06<00:02,  3.85it/s]\u001b[A\n",
      " 74%|███████▎  | 28/38 [00:07<00:02,  4.11it/s]\u001b[A\n",
      " 76%|███████▋  | 29/38 [00:07<00:02,  4.14it/s]\u001b[ALoss = 1.0802e-02, PNorm = 55.6425, GNorm = 0.5122, lr_0 = 1.0000e-04\n",
      "\n",
      " 79%|███████▉  | 30/38 [00:07<00:01,  4.04it/s]\u001b[A\n",
      " 82%|████████▏ | 31/38 [00:07<00:01,  3.83it/s]\u001b[A\n",
      " 84%|████████▍ | 32/38 [00:08<00:01,  3.84it/s]\u001b[A\n",
      " 87%|████████▋ | 33/38 [00:08<00:01,  3.97it/s]\u001b[A\n",
      " 89%|████████▉ | 34/38 [00:08<00:01,  3.78it/s]\u001b[A\n",
      " 92%|█████████▏| 35/38 [00:09<00:00,  3.61it/s]\u001b[A\n",
      " 95%|█████████▍| 36/38 [00:09<00:00,  3.91it/s]\u001b[A\n",
      " 97%|█████████▋| 37/38 [00:09<00:00,  4.00it/s]\u001b[A\n",
      "100%|██████████| 38/38 [00:09<00:00,  4.37it/s]\u001b[A\n",
      "                                               \u001b[A\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]\u001b[A\n",
      " 40%|████      | 2/5 [00:00<00:00, 12.45it/s]\u001b[A\n",
      " 80%|████████  | 4/5 [00:00<00:00, 12.07it/s]\u001b[A\n",
      "                                             \u001b[AValidation auc = 0.831169\n",
      "100%|██████████| 30/30 [05:00<00:00, 10.01s/it]\n",
      "Model 0 best validation auc = 0.832251 on epoch 12\n",
      "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
      "Loading pretrained parameter \"ffn.1.weight\".\n",
      "Loading pretrained parameter \"ffn.1.bias\".\n",
      "Loading pretrained parameter \"ffn.4.weight\".\n",
      "Loading pretrained parameter \"ffn.4.bias\".\n",
      "Loading pretrained parameter \"ffn.7.weight\".\n",
      "Loading pretrained parameter \"ffn.7.bias\".\n",
      "Model 0 test auc = 0.648485                  \n",
      "Ensemble test auc = 0.648485\n",
      "1-fold cross validation\n",
      "\tSeed 0 ==> test auc = 0.648485\n",
      "Overall test auc = 0.648485 +/- 0.000000\n",
      "Elapsed time = 0:07:25\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train a binary classifier Chemprop model using the hyperparameters and training data specified, using the env\n",
    "cmd = 'chemprop_train --data_path notebook_files/train.csv --dataset_type classification --config_path notebook_files/hyperparameters.json --save_dir \"notebook_files/checkpoint\" --ensemble_size 1 --features_generator rdkit_2d_normalized --no_features_scaling'\n",
    "print(cmd)\n",
    "subprocess.call(cmd, shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chemprop_predict --test_path notebook_files/test.csv --checkpoint_dir \"notebook_files/checkpoint\" --preds_path \"notebook_files/predictions.csv\"  --features_generator rdkit_2d_normalized --no_features_scaling\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "266it [00:00, 5171.67it/s]\n",
      "100%|██████████| 266/266 [00:12<00:00, 21.21it/s]\n",
      "  0%|          | 0/1 [00:00<?, ?it/s]\n",
      "  0%|          | 0/6 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading training args\n",
      "Setting molecule featurization parameters to default.\n",
      "Loading data\n",
      "Validating SMILES\n",
      "Test size = 266\n",
      "Predicting with an ensemble of 1 models\n",
      "Loading pretrained parameter \"encoder.encoder.0.cached_zero_vector\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_i.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_h.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_o.weight\".\n",
      "Loading pretrained parameter \"encoder.encoder.0.W_o.bias\".\n",
      "Loading pretrained parameter \"ffn.1.weight\".\n",
      "Loading pretrained parameter \"ffn.1.bias\".\n",
      "Loading pretrained parameter \"ffn.4.weight\".\n",
      "Loading pretrained parameter \"ffn.4.bias\".\n",
      "Loading pretrained parameter \"ffn.7.weight\".\n",
      "Loading pretrained parameter \"ffn.7.bias\".\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "WARNING:root:No normalization for BCUT2D_MWHI\n",
      "WARNING:root:No normalization for BCUT2D_MWLOW\n",
      "WARNING:root:No normalization for BCUT2D_CHGHI\n",
      "WARNING:root:No normalization for BCUT2D_CHGLO\n",
      "WARNING:root:No normalization for BCUT2D_LOGPHI\n",
      "WARNING:root:No normalization for BCUT2D_LOGPLOW\n",
      "WARNING:root:No normalization for BCUT2D_MRHI\n",
      "WARNING:root:No normalization for BCUT2D_MRLOW\n",
      "\n",
      " 17%|█▋        | 1/6 [00:17<01:28, 17.64s/it]\u001b[A\n",
      " 33%|███▎      | 2/6 [00:17<00:49, 12.38s/it]\u001b[A\n",
      " 50%|█████     | 3/6 [00:17<00:26,  8.70s/it]\u001b[A\n",
      " 67%|██████▋   | 4/6 [00:17<00:12,  6.12s/it]\u001b[A\n",
      "100%|██████████| 1/1 [00:48<00:00, 48.21s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving predictions to notebook_files/predictions.csv\n",
      "Elapsed time = 0:01:01\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the trained Chemprop model to make predictions on test.csv\n",
    "cmd = 'chemprop_predict --test_path notebook_files/test.csv --checkpoint_dir \"notebook_files/checkpoint\" --preds_path \"notebook_files/predictions.csv\"  --features_generator rdkit_2d_normalized --no_features_scaling'\n",
    "print(cmd)\n",
    "subprocess.call(cmd, shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of compounds whose scores were predicted: 266\n"
     ]
    }
   ],
   "source": [
    "# Read in predictions for performing downstream analyses\n",
    "PRED_SMILES = []\n",
    "PRED_SCORES = []\n",
    "for file in glob.glob(\"./notebook_files/predictions.csv\"):\n",
    "    fopen = open(file);\n",
    "    for line in fopen.readlines():\n",
    "        y = [value for value in line.split('\\n')]\n",
    "        y = y[0].split(\",\")\n",
    "        try: \n",
    "            PRED_SMILES.append(y[0])\n",
    "            PRED_SCORES.append(float(y[1]))\n",
    "        except:\n",
    "            x=1;\n",
    "    fopen.close()\n",
    "print('Number of compounds whose scores were predicted:',len(PRED_MOLS))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgQklEQVR4nO3deXRV5b3/8fc3J/MckhBCJuZRQSCAgAjWCRyuVbHigEO1FKdae+vVq+3tdH+92uG21rFoqVOrthStWIeqCCqTJMg8SJhDwIQpjCHT8/sjuTRggAOcZJ9z8nmt5VqcszfnfB7D+rDZw/OYcw4REQl9EV4HEBGRwFChi4iECRW6iEiYUKGLiIQJFbqISJiI9OqLMzIyXKdOnbz6ehGRkFRcXLzdOZfZ3DbPCr1Tp04UFRV59fUiIiHJzDYea5tOuYiIhAkVuohImFChi4iECRW6iEiYUKGLiIQJFbqISJhQoYuIhImQK/SS8r38dPoKqmvrvY4iIhJUQq7QN+88yJTZ65m5utzrKCIiQSXkCn1k9wwyEqOZtnCL11FERIKKX4VuZmPMbLWZlZjZg81sTzGz6Wa22MyWm9mtgY/aINIXweX9OzJjVTmVB2pa6mtERELOCQvdzHzAk8BYoA9wnZn1OWq3u4AVzrn+wGjg12YWHeCsh40blEt1XT0PvbGU+notoSciAv4doQ8BSpxz65xz1cCrwBVH7eOAJDMzIBHYCdQGNGkTfTum8NAlvfjHkq08PWttS32NiEhI8afQc4DNTV6XNr7X1BNAb6AMWArc65z7ym0oZjbRzIrMrKiiouIUIzeYeG5Xxp7RgcdnrKFs98HT+iwRkXDgT6FbM+8dfZ7jYmAR0BE4C3jCzJK/8pucm+ycK3TOFWZmNjud70l5+NLeAHzvL4uoqqk77c8TEQll/hR6KZDX5HUuDUfiTd0KTHMNSoD1QK/ARDy23LR4HrmqH/PW7eRbLxaxc391S3+liEjQ8qfQFwDdzaxz44XO8cCbR+2zCTgfwMyygJ7AukAGPZavD8jhF1f3Y/66nVz82495Z+nW1vhaEZGgc8JCd87VAncD7wErgb8455ab2SQzm9S428+A4Wa2FPgQeMA5t72lQh/tG4PzmHbncLKSY7jjTwv5cOWXrfXVIiJBw5zz5ra/wsJCF+gl6Kpq6rjqqTlsrTzIe989l/bJsQH9fBERr5lZsXOusLltIfek6PHERvl4/PoB7K+u46dvrfA6johIqwqrQgfompnIXaO78daSrTr1IiJtStgVOsCk0V3o1SGJ+6cuYVtllddxRERaRVgWekykjyeuH8jB6joenLYEr64TiIi0prAsdIBu7RO5/+KezFxdwZ/mb/I6johIiwvbQge4eXgnhndN5wdvLOPnb6+ktk6LYohI+ArrQvdFGH+8dTATzi5g8sfruOWPC7TSkYiErbAudGg4n/6zr5/BI1edyacl23nknVVeRxIRaRGRXgdoLeOH5LNq216mzF7PkM5pjDkj2+tIIiIBFfZH6E09dElv+uel8v2/LuG/31rB1kpNuysi4aNNFXp0ZARP3TCQs/JSeXHuRi5/fDbPz15Pxd5DXkcTETltbarQAXJS43j59qH84zvnkBofxY+nr+CmKZ9pKTsRCXltrtD/T/esJN6/71x+fU1/Vm7dw5uLj57iXUQktLTZQgcwM64ckEOf7GS++9oirn92nu5VF5GQ1aYLHSAiwphyy2C+OaIzc9bu4IOV5V5HEhE5JW2+0AE6pMTy0CW96JgSy0vzNmjuFxEJSSr0RpG+CK4fms/skh0M+u8PWLal0utIIiInRYXexLdHdeWRq84E4BfvrfY4jYjIyVGhNxHli2D8kHwmntuFj7+oYNYXFV5HEhHxmwq9GTcNK6BLZgK3Pb+A52ev1zl1EQkJKvRmxEdH8vqdIxjdM5MfT1/BHS8vZPPOA17HEhE5LhX6MaTERTF5QiEPjOnFrC8quOR3n7Bw0y6vY4mIHJMK/TgiIow7Rnfln/edS3pCNBOem8/8dTu8jiUi0iwVuh/y2sXz2reH0SEllpv/+Bkl5fu8jiQi8hUqdD9lJcfyyrfOJioigp+/vdLrOCIiX9FmFrgIhPbJsdxzfjd+/vYqBv+/DxjdI5PBndrRLiGaThkJZCbGkBIf5XVMEWmjVOgn6ZbhnXEOVm7dw1tLtvLX4tIjtnfJSODZmwvpmpnoUUIRaatU6CcpOjKCb4/qCsD+Q7Xs3F9N+d4qNu88yJd7qnhm1lq+99oifnfdANISokmKicTMPE4tIm2BCv00JMREkhATSV67eAYVNLyX1y6eO/+0kFG/nAnA4E5p3H9xL3p2SCIlTqdjRKTlmFdPQRYWFrqioiJPvrulLdy0i/UV+ynbfZBnP1nHnqpaMhKjef++UaQlRHsdT0RCmJkVO+cKm9umI/QWMDA/jYH5aQBcNzSfT9ds53t/WcQTH5Xww8v6eJxORMKVbltsYRmJMXx9QA7jBuXy0tyNLNiw0+tIIhKmdITeSh4c25uiDbu4ecpn9MtNoV1CNGnx0WSnxNIvN5XhXdOJ9OnvVxE5dSr0VtIuIZqXbx/Ko++uYuvuKr74ch8791ezc381AFnJMVw5IJebhhXQMTXO47QiEop0UdRj+w/V8smaCqYWl/LR6goMuO/CHtwxqisREbrdUUSOdLyLoir0ILJl90H+5+2VvLVkK/1yU7jvgh6M6pGpYheRw45X6DppG0RyUuN4/LoBPDb+LLbvPcStzy/ghufmc6i2zutoIhICVOhBxsy44qwcZt5/Hj/5t77MXbeDu//8OZt2aIENETk+XRQNUtGREdw8vBM1dfU8+u4qZn1RwS/H9WN41wxioyJIitVTpyJyJL/OoZvZGOAxwAc855x7pJl9RgO/BaKA7c65Ucf7TJ1D99+2yiru+vNCijf+a8Wkn13RlwnDOnkXSkQ8cVoXRc3MB3wBXAiUAguA65xzK5rskwrMAcY45zaZWXvnXPnxPleFfnKqaup4b/k29lTV8tLcDUSY8e53z/U6loi0stN99H8IUOKcW9f4Ya8CVwArmuxzPTDNObcJ4ERlLicvNsrHFWflAFBf7/jRm8tZtW0PvToke5xMRIKFPxdFc4DNTV6XNr7XVA8gzcxmmlmxmd3U3AeZ2UQzKzKzooqKilNLLFzaLxtfhDHppWJenrfR6zgiEiT8KfTmboI++jxNJDAIuBS4GPihmfX4ym9ybrJzrtA5V5iZmXnSYaVBRmIMD4zpSZQvgp9MX075niqvI4lIEPCn0EuBvCavc4GyZvZ51zm33zm3HfgY6B+YiNKcied25dmbCqmtd/xh9nq8ekBMRIKHP+fQFwDdzawzsAUYT8M586b+DjxhZpFANDAU+E0gg8pXdcpI4KI+Wfx+1jpenruRbllJdG+fSJ/sZEb3zKSLlsETaVNOWOjOuVozuxt4j4bbFqc455ab2aTG7c8451aa2bvAEqCehlsbl7VkcGnwq2v6M7J7GSXl+1hTvpeZqxvmhfnpW1CQHk/vDsmckZPMeb3a07djitdxRaQFaS6XMFS2+yBvL93Kwk27WLV1L+t37Afg+VuHMKqHrl2IhDJNztXG7dh3iOuencfO/TXcOborGUkx9MhKpGdWkhawFgkxWoKujUtPjOF31w3g9heK+Olb/3p8ICMxmuFdMxjRLZ1zumeSo3nYRUKajtDbmIq9h9h1oJrFm3czZ+0OPi3ZTsXeQ5jB+MF53H9xL9ppIWuRoKVTLnJMzjnWlO/jlc828eLcjSRE+3j+m0MOL3ItIsFF86HLMZkZPbKS+NHlfXn33pEAvDJ/k8epRORUqNDlsO5ZSYzolsHsku16UEkkBKnQ5QjDu2VQVlnFBi2oIRJyVOhyhBFd0wGYuVoTZoqEGhW6HKFzRgI9shL56VsrGPf0HJ6ZtdbrSCLiJxW6HMHMeG3iMO4a3Y2DNXU88s4qPtLRukhIUKHLV6QlRPP9i3vy+p0j6JqZwEPTlvKn+RtZ8+VeXSwVCWIqdDmm6MgIfv2Ns4iP9vHw68u48DcfM/IXH/H20q1eRxORZujBIjkh5xxrK/ZRvHEXf5q/iSWllfxyXD+uKcw78W8WkYDSg0VyWsyMbu2TuHZwPn/59jAG5Kfy2IdrqK/X6ReRYKJCl5MSG+XjpmEFlO46yGcbdnodR0SaUKHLSbu4bwcSon28MGeDjtJFgoimz5WTFh8dyS0jOvHkR2u56Lcf0yE5lrhoH8mxUdw8vIB+ualeRxRpk1Tockq+f1FPCtIT+PuiLeyvrmX7vkNsraxi+uIyLu2XzbWD8zi7S7rXMUXaFN3lIgGza381//2PlXy0upx9VbX8/KozubhvFkmxUV5HEwkbmg9dWlXlgRpu/MN8lm6pJDYqguuHFPAfY3oSG+XzOppIyNMSdNKqUuKjmHrHMIo37OL1z7cwZfZ6NuzYz/9+oz+p8VoNSaSlqNClRcRE+hjeLYPh3TLon5fKD95YxtCff0h2Sizx0ZHER/vITYvj3B6Z9M5Opnd2steRRUKeCl1a3I1nFzCoII2pxaWU7z3Ewepa9h+q45M123ljURkAL982lHO6Z3icVCS0qdClVfTOTuaHl/U54r3aunrWbd/PrX9cwC/eW8WIbiMwM48SioQ+PVgknon0RdAjK4l7L+jOktJKZq6u8DqSSEhToYvnrhyQQ7uEaKYuLPU6ikhIU6GL56J8EVzWL5sPVnzJ3qoar+OIhCwVugSFrw/I4VBtPTc+N59/LNF86yKnQhdFJSgMyEvluxd0583FZdz154V8sDKH9IRoCjISuGFIPhERulgqciIqdAkKZsZ3L+jBpFFdefj1Zcz6opwD1XUcqK5jW+VB/v3Cnip1kRPQo/8StJxzPPi3pbxWtJmUuCi+f3FPbhyar1sbpU3TXC4Ssmrq6pm+uIy/LSxldskOOqbEkp8eT3JsFJ0zErhlRCeyU+K8jinSalToEvLq6x1Ti0v5tGQ72yqr2FNVQ0n5PiLMuH5oPg9d0pvoSF3jl/Cnybkk5EVEGN8YnMc3Bv9rYerNOw/wxIwSnp+zgd7ZDWueirRlOqSRkJXXLp5Hrj6TzhkJvP75Fq/jiHhOhS4hzcy4ckAO89btpHjjLrw6hSgSDFToEvKuHJBDbFQEVz89h4deX+p1HBHP6By6hLy8dvF88L1RPPvxOl6Yu5GYSB83nl1At/aJXkcTaVU6QpewkJsWz39d3pdLz8zmhbkbGD95LtW19V7HEmlVfhW6mY0xs9VmVmJmDx5nv8FmVmdm4wIXUcQ/vgjjyRsG8txNhWzfV82MVeVeRxJpVScsdDPzAU8CY4E+wHVm1ucY+z0KvBfokCInY1SPTNonxTC1eLPXUURalT9H6EOAEufcOudcNfAqcEUz+90D/A3QYZF4KtIXwbhBucxYVU7xxp1exxFpNf4Ueg7Q9FCntPG9w8wsB7gSeCZw0URO3R2ju9IxNY7vvraIA9W1XscRaRX+FHpzMyEdfbPvb4EHnHN1x/0gs4lmVmRmRRUVWm5MWk5SbBS/HNefzTsP8uf5m7yOI9Iq/Cn0UiCvyetcoOyofQqBV81sAzAOeMrMvn70BznnJjvnCp1zhZmZmaeWWMRPw7qmM6xLOpM/XkdVzXGPNUTCgj+FvgDobmadzSwaGA+82XQH51xn51wn51wnYCpwp3PujUCHFTlZ3zm/O+V7D/HMrLVeRxFpcSd8sMg5V2tmd9Nw94oPmOKcW25mkxq367y5BK1hXdP5+lkdeWJGCekJ0Vw5MJfEGD1PJ+FJ0+dK2Nt9oJrxk+exatteBuSn8urEs4mJ9HkdS+SUHG/6XD0pKmEvNT6ad+4dyWPjz+LzTbu558+fU3mgxutYIgGnQpc2wcy44qwcfnBpb2asKufC38zio9V6ZELCiwpd2pTbR3bhjbtGkBYfze0vFLGuYp/XkUQCRoUubc4ZOSm8fPtQIiOMp2bq7hcJHyp0aZMyk2K4YWgB0xaWcvnjn7Jq2x6vI4mcNhW6tFn3XtCdb53bhdJdB3j49WVa7UhCngpd2qyUuCj+c2xvHhzbi+KNu7j9hSLeWbqV+noVu4QmPWEhbd64QXms2raXd5Zu48NVC+nbMZkfXNqHYV3TvY4mclJ0hC5tni/C+NHlfZn94Nf49TX9qTxYw63Pf0blQd2rLqFFhS7SyBdhXD0ol6dvGERVTT1/X7TF60giJ0WFLnKUM3NT6NsxmVc+26wLpRJSVOgizZhwdgErt+7h9c91lC6hQ4Uu0oxrCvMY3CmNH725nLLdB72OI+IXFbpIM3wRxq+u6U9dveM/pi7RrYwSElToIsdQkJ7AQ5f05tOS7fT7yT95YOoStuhoXYKY7kMXOY4bhuaTFBvJ7JLtvL5oC28uLuOHl/Xh+qH5XkcT+QodoYscx/9Nu/uLcf2Z8e+jKOyUxsNvLGVOyXavo4l8hQpdxE+5afH8fsIgumQkcO9ri9i+75DXkUSOoEIXOQnx0ZE8cf1AKg/W8P2/LtbFUgkqKnSRk9Q7O5kfXtaHmasr+MOn672OI3KYCl3kFNw4NJ+xZ3Tg0XdXUVKuVY8kOKjQRU6BmfHTK86g3jmmLy7zOo4IoEIXOWWZSTEUFrTjnyu+9DqKCKBCFzktF/XNYuXWPWzeecDrKCIqdJHTcVGfDvgijKuensMfPl1PTV2915GkDVOhi5yG/PR4Xr5tKD2yEvnZWyuY9FKxptwVz6jQRU7TsK7pvHzbUB4Y04sPV5UztbhU96eLJ1ToIgFgZkw8twv9clO4f+oSRv7iIxZv3u11LGljVOgiAeKLMF765lAevfpMzODayXPZuGO/17GkDVGhiwRQSnwU1w7O56+ThlHv4OmZa72OJG2ICl2kBWSnxDF+cB5/W1jK+u06SpfWoUIXaSGTRnUlLsrHzVM+0zJ20ipU6CItpGNqHC/eNpSd+6v5tydms7ys0utIEuZU6CIt6Ky8VKbdOZzq2jr+8IlmZpSWpUIXaWE9spLolZ1M6S6ddpGWpUIXaQW5qXGU7tJ8L9KyVOgirSA3LY5te6o014u0KBW6SCvISYuj3sG2yiqvo0gYU6GLtILctHgAnUeXFqVCF2kFuWlxADqPLi3Kr0I3szFmttrMSszswWa232BmSxr/m2Nm/QMfVSR0ZafEYQZb9ICRtKATFrqZ+YAngbFAH+A6M+tz1G7rgVHOuX7Az4DJgQ4qEsqiIyPISorVKRdpUf4coQ8BSpxz65xz1cCrwBVNd3DOzXHO7Wp8OQ/IDWxMkdCX1y6ONeX7vI4hYcyfQs8BNjd5Xdr43rHcBrzT3AYzm2hmRWZWVFFR4X9KkTBwbvdMFm/erXldpMX4U+jWzHvNLsdiZufRUOgPNLfdOTfZOVfonCvMzMz0P6VIGLisf0cA3l661eMkEq4i/dinFMhr8joXKDt6JzPrBzwHjHXO7QhMPJHw0Tkjgb4dk/nj7A3sP1RHl8wELjkzG19Ec8dMIifPnyP0BUB3M+tsZtHAeODNpjuYWT4wDZjgnPsi8DFFwsODY3uRGBPJbz74gnte+Zxf/3O115EkjJzwCN05V2tmdwPvAT5ginNuuZlNatz+DPBfQDrwlJkB1DrnClsutkhoGtk9k/fuy6Sqpo6fTF/OUzPXck63DIZ3y/A6moQBc86b1ckLCwtdUVGRJ98tEgwO1dYx/H9mMLhTO56ZMMjrOBIizKz4WAfMelJUxCMxkT6uGpjDByu/ZPu+Q17HkTCgQhfx0LWD86itd1z4v7N4ad5Gr+NIiFOhi3ioW/sknrx+IOmJMfzhk3V4dQpUwoMKXcRjl/bL5ubhndiw4wBrK/QkqZw6FbpIELigd3sA3l9R7nESCWUqdJEgkJ0Sx5k5KbyzTE+RyqlToYsEiasH5rCktJKlpZVeR5EQpUIXCRJXDcolLsrH83M2eB1FQpQ/c7mISCtIjo3imsJcXpy7kT1VNdw6ohPDuqTT+PS1yAmp0EWCyMOX9iYrOZbfz1rL+yu+ZGT3DH51TX+ykmO9jiYhQKdcRIJITKSPu87rxmcPX8CPLu9D8cZd3PbCAtZV7KO6tt7reBLkVOgiQSg2ysetIzrz+HUDWF62h6/9ehZ3/Xmh17EkyKnQRYLY+b2zeOPOEVxbmMf7K75k2RbdASPHpkIXCXL981J5+LLeJMVG8uRHJV7HkSCmQhcJAcmxUdx4dgHvLd/GFq1JKsegQhcJEdcPyccBr362yesoEqRU6CIhIq9dPOf1bM+LczcyfXEZtXW660WOpEIXCSEPX9qbnNQ47nnlc8559CPNzihHUKGLhJCumYm8efcInrlxIPsO1fKr97TItPyLnhQVCTGRvgjGnJHNirI9/G5GCT94Y+nh2RoHFqSREO3TdAFtlApdJETddk4XPinZzt8/L2PvodrD7xekx/O3O4aTkRjjYTrxggpdJESlxEfx+p0jANhbVcOCDTtZuXUvj324hv+ctpSnbxhIpE9nVdsS/bRFwkBSbBRf65XFXed14/6LevL+ii85/39nUVK+1+to0opU6CJh5vaRnZk8YRD7qmq599VFmtSrDVGhi4QZM+Oivh34+VVnsrxsD7+ftdbrSNJKVOgiYerivh0Ye0YHnpq5lq2Vmi6gLVChi4Sxhy7pTZ1zPPrOKq+jSCtQoYuEsbx28Uwc2YU3FpVRvHGX13GkhanQRcLcHaO7kpUcw6SXi3lm1lpmrPoS55zXsaQFqNBFwlxCTCRTbhlMdkosj7yzim8+X8R9ry3S5F5hSA8WibQBfTum8Pe7RrBzfzUvzt3IYx+u4fzeWVzev6PX0SSAdIQu0kaYGemJMdx7fndyUuP4a3Gp15EkwFToIm1MRIRx9cAcPllTwUvzNrK8rFIPH4UJnXIRaYOuKczj2U/W88M3lh1+Ly0+it7ZyZyZm8Ll/TrSt2OyZm0MMebV1e7CwkJXVFTkyXeLCFTV1FG+5xALNuykbPdByioPsqJsDyu27qGmztEjK5HuWUmkxkWRFh9NanwUBekJnNczU5N+ecjMip1zhc1t0xG6SBsVG+UjPz2e/PT4I97ffaCa6Uu28vaSraws28PugzXsPlBNfeOxX25aHLef05kJwzrhi9ARfDBRoYvIEVLjo5lwdgETzi44/F59vWPvoVrmr9vB5I/X8ePpK9hTVct3zu/uYVI5mv7dJCInFBFhpMRFcVHfDky9YziX9+/I4zPWMHN1uR5SCiI6QheRk/bjy/uwaPMubvnjAjqmxNIvN5W+HZPJbRdHalw0I7plEB2p48XW5lehm9kY4DHABzznnHvkqO3WuP0S4ABwi3NuYYCzikiQSE+M4f37RjFt4RbmrN3OktJK3l2+7fD2jMRoenZI4oLeWfxb/46kazm8VnHCu1zMzAd8AVwIlAILgOuccyua7HMJcA8NhT4UeMw5N/R4n6u7XETCy4HqWrZVVrFhx36mLdxCSfk+Vm1rWDEpOyWWgQVpXNQni8ykGFLiokiNjyYlLkqLWp+k073LZQhQ4pxb1/hhrwJXACua7HMF8KJr+Nthnpmlmlm2c27raWYXkRARHx1Jl8xEumQm8rVeWQAsKd3N/HU7WVZWyeyS7fxjyVcrITLCSI6LIiHGh88MM8MMIswI15q/dnAet4/sEvDP9afQc4DNTV6X0nAUfqJ9coAjfnpmNhGYCJCfn3+yWUUkxPTLTaVfbioAtXX1rCnfx+4DNVQerKbyYA2VB2saX9ew/1AtDqh34JyjPowvtma00Ckofwq9ub8kj/4/7c8+OOcmA5Oh4ZSLH98tImEi0hdB7+xkr2OENX8uQ5cCeU1e5wJlp7CPiIi0IH8KfQHQ3cw6m1k0MB5486h93gRusgZnA5U6fy4i0rpOeMrFOVdrZncD79Fw2+IU59xyM5vUuP0Z4G0a7nApoeG2xVtbLrKIiDTHr/vQnXNv01DaTd97psmvHXBXYKOJiMjJ0KNcIiJhQoUuIhImVOgiImFChS4iEiY8W7HIzCqAjaf42zOA7QGMEyra4rg15rZBY/ZfgXMus7kNnhX66TCzomNNThPO2uK4Nea2QWMODJ1yEREJEyp0EZEwEaqFPtnrAB5pi+PWmNsGjTkAQvIcuoiIfFWoHqGLiMhRVOgiImEiqAvdzMaY2WozKzGzB5vZbmb2u8btS8xsoBc5A8mPMd/QONYlZjbHzPp7kTOQTjTmJvsNNrM6MxvXmvlagj9jNrPRZrbIzJab2azWzhhofvzZTjGz6Wa2uHHMIT9rq5lNMbNyM1t2jO2B7TDnXFD+R8NUvWuBLkA0sBjoc9Q+lwDv0LBi0tnAfK9zt8KYhwNpjb8e2xbG3GS/GTTM+jnO69yt8HNOpWHd3vzG1+29zt0KY34IeLTx15nATiDa6+ynOe5zgYHAsmNsD2iHBfMR+uHFqZ1z1cD/LU7d1OHFqZ1z84BUM8tu7aABdMIxO+fmOOd2Nb6cR8PqUKHMn58zwD3A34Dy1gzXQvwZ8/XANOfcJgDnXKiP258xOyDJzAxIpKHQa1s3ZmA55z6mYRzHEtAOC+ZCP9bC0ye7Tyg52fHcRsPf7qHshGM2sxzgSuAZwoM/P+ceQJqZzTSzYjO7qdXStQx/xvwE0JuG5SuXAvc65+pbJ55nAtphfi1w4ZGALU4dQvwej5mdR0Ohn9OiiVqeP2P+LfCAc66u4eAt5Pkz5khgEHA+EAfMNbN5zrkvWjpcC/FnzBcDi4CvAV2B983sE+fcnhbO5qWAdlgwF3pbXJzar/GYWT/gOWCsc25HK2VrKf6MuRB4tbHMM4BLzKzWOfdGqyQMPH//bG93zu0H9pvZx0B/IFQL3Z8x3wo84hpOLpeY2XqgF/BZ60T0REA7LJhPubTFxalPOGYzywemARNC+GitqROO2TnX2TnXyTnXCZgK3BnCZQ7+/dn+OzDSzCLNLB4YCqxs5ZyB5M+YN9HwLxLMLAvoCaxr1ZStL6AdFrRH6K4NLk7t55j/C0gHnmo8Yq11ITxLnZ9jDiv+jNk5t9LM3gWWAPXAc865Zm99CwV+/px/BjxvZktpOBXxgHMupKfUNbNXgNFAhpmVAj8CoqBlOkyP/ouIhIlgPuUiIiInQYUuIhImVOgiImFChS4iEiZU6CIiYUKFLiISJlToIiJh4v8DRmuBDRqXyQMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot all the prediction scores, rank-ordered\n",
    "indi = numpy.argsort(PRED_SCORES)\n",
    "sor = numpy.take(PRED_SCORES,indi)\n",
    "xx = [x/len(sor) for x in range(len(sor))]\n",
    "plt.plot(xx,sor[::-1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chemprop",
   "language": "python",
   "name": "chemprop"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}