Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We have packaged the whole training procedure into this Hyper fit funciton. It fits many different models and selects the best one and saves it\n",
    "# training process can be inspected by calling:\n",
    "#tensorboard --logdir=$MODEL_DIR --port=8080 in shell \n",
    "#Replace log dir with whatever you named it and also replace the IP \n",
    "\n",
    "#You should not run this in a notebook but copy it to shell.\n",
    "#Also you should restart the server/ shut down all other notebook before starting the training. \n",
    "#Currently the model takes ca. 4 days for training 20 iteraitons of it. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/leander/anaconda3/envs/pytorch/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/home/leander/anaconda3/envs/pytorch/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "from Fit_Function import Hyper_fit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at provided point.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, None, 100)         1220200   \n",
      "_________________________________________________________________\n",
      "bidirectional (Bidirectional (None, None, 156)         112320    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, None, 156)         0         \n",
      "_________________________________________________________________\n",
      "multi__attention (Multi_Atte [(None, 128, 20), (None,  42753     \n",
      "_________________________________________________________________\n",
      "loop__projection (Loop_Proje (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization (BatchNo (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "loop__projection_1 (Loop_Pro (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization_2 (Batch (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "loop__projection_2 (Loop_Pro (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "permute (Permute)            (None, 20, 128)           0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 20, 13598)         1754142   \n",
      "_________________________________________________________________\n",
      "sum_dum (sum_dum)            (None, 13598)             0         \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 13598)             0         \n",
      "=================================================================\n",
      "Total params: 4,121,911\n",
      "Trainable params: 4,121,022\n",
      "Non-trainable params: 889\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/leander/anaconda3/envs/pytorch/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:112: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "391/391 [==============================] - 85s 216ms/step - loss: 0.4885 - acc: 0.7458 - val_loss: 0.0422 - val_acc: 0.9998\n",
      "Epoch 2/50\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.0106 - acc: 0.99990.33325574427553883\n",
      "391/391 [==============================] - 948s 2s/step - loss: 0.0106 - acc: 0.9999 - val_loss: 0.0015 - val_acc: 0.9999\n",
      "Epoch 3/50\n",
      "390/391 [============================>.] - ETA: 0s - loss: 0.0015 - acc: 1.00000.3327394334719624\n",
      "391/391 [==============================] - 911s 2s/step - loss: 0.0015 - acc: 1.0000 - val_loss: 7.2768e-04 - val_acc: 0.9999\n",
      "0.33325574427553883\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, None, 100)         1220200   \n",
      "_________________________________________________________________\n",
      "bidirectional_1 (Bidirection (None, None, 156)         111696    \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, None, 156)         0         \n",
      "_________________________________________________________________\n",
      "multi__attention_1 (Multi_At [(None, 128, 20), (None,  42753     \n",
      "_________________________________________________________________\n",
      "loop__projection_3 (Loop_Pro (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_7 (Batch (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "loop__projection_4 (Loop_Pro (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization_8 (Batch (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation_5 (Activation)    (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_9 (Batch (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "loop__projection_5 (Loop_Pro (None, 128, 20)           330240    \n",
      "_________________________________________________________________\n",
      "batch_normalization_10 (Batc (None, 128, 20)           80        \n",
      "_________________________________________________________________\n",
      "activation_6 (Activation)    (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_11 (Batc (None, 128, 20)           512       \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 128, 20)           0         \n",
      "_________________________________________________________________\n",
      "permute_1 (Permute)          (None, 20, 128)           0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 20, 13598)         1754142   \n",
      "_________________________________________________________________\n",
      "sum_dum_1 (sum_dum)          (None, 13598)             0         \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 13598)             0         \n",
      "=================================================================\n",
      "Total params: 4,121,287\n",
      "Trainable params: 4,120,398\n",
      "Non-trainable params: 889\n",
      "_________________________________________________________________\n",
      "Iteration No: 1 ended. Evaluation done at provided point.\n",
      "Time taken: 1968.1193\n",
      "Function value obtained: -0.3333\n",
      "Current minimum: -0.3333\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, None)              0         \n",
      "_________________________________________________________________\n",
      "embedding (Embedding)        (None, None, 86)          1049372   \n",
      "_________________________________________________________________\n",
      "conv1d (Conv1D)              (None, None, 118)         30562     \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, None, 118)         0         \n",
      "_________________________________________________________________\n",
      "multi__attention (Multi_Atte [(None, 110, 48), (None,  60185     \n",
      "_________________________________________________________________\n",
      "permute (Permute)            (None, 48, 110)           0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 48, 13598)         1509378   \n",
      "_________________________________________________________________\n",
      "sum_dum (sum_dum)            (None, 13598)             0         \n",
      "_________________________________________________________________\n",
      "activation (Activation)      (None, 13598)             0         \n",
      "=================================================================\n",
      "Total params: 2,649,497\n",
      "Trainable params: 2,649,496\n",
      "Non-trainable params: 1\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      "391/391 [==============================] - 78s 200ms/step - loss: 0.1747 - acc: 0.9423 - val_loss: 9.1492e-04 - val_acc: 0.9999\n",
      "Epoch 2/50\n",
      "390/391 [============================>.] - ETA: 0s - loss: 7.1529e-04 - acc: 0.99990.33823139971449684\n",
      "391/391 [==============================] - 941s 2s/step - loss: 7.1512e-04 - acc: 0.9999 - val_loss: 6.2438e-04 - val_acc: 1.0000\n",
      "Epoch 3/50\n",
      "390/391 [============================>.] - ETA: 0s - loss: 5.7311e-04 - acc: 1.0000"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-1f229871f0bd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     49\u001b[0m           \u001b[0;34m,\u001b[0m\u001b[0msave_cpu\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m           ,batch_size=128)\n\u001b[0m",
      "\u001b[0;32m~/AI/noovle-open-source/Fit_Function.py\u001b[0m in \u001b[0;36mHyper_fit\u001b[0;34m(path_best_model, TB_logs, steps, validation_steps, train_files, validation_files, rounds, label_tokenizer_path, text_tokenizer_path, max_epoch, buffer_size, default_parameters, batch_size, evaluation, single_fit, baseline, interval, wait_epochs, eval_size, save_cpu)\u001b[0m\n\u001b[1;32m    326\u001b[0m                                     \u001b[0mn_calls\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrounds\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    327\u001b[0m                                     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 328\u001b[0;31m                                     x0=default_parameters)\n\u001b[0m\u001b[1;32m    329\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m         \u001b[0mplot_convergence\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msearch_result\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/skopt/optimizer/gp.py\u001b[0m in \u001b[0;36mgp_minimize\u001b[0;34m(func, dimensions, base_estimator, n_calls, n_random_starts, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, noise, n_jobs)\u001b[0m\n\u001b[1;32m    226\u001b[0m         \u001b[0mn_restarts_optimizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_restarts_optimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    227\u001b[0m         \u001b[0mx0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my0\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0my0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrng\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 228\u001b[0;31m         callback=callback, n_jobs=n_jobs)\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/skopt/optimizer/base.py\u001b[0m in \u001b[0;36mbase_minimize\u001b[0;34m(func, dimensions, base_estimator, n_calls, n_random_starts, acq_func, acq_optimizer, x0, y0, random_state, verbose, callback, n_points, n_restarts_optimizer, xi, kappa, n_jobs)\u001b[0m\n\u001b[1;32m    246\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mn\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_calls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m         \u001b[0mnext_x\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m         \u001b[0mnext_y\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_x\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    249\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnext_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnext_y\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    250\u001b[0m         \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mspecs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mspecs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/skopt/utils.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    634\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m             \u001b[0;31m# Call the wrapped objective function with the named arguments.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 636\u001b[0;31m             \u001b[0mobjective_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0marg_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    637\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    638\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mobjective_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/AI/noovle-open-source/Fit_Function.py\u001b[0m in \u001b[0;36mfit_model\u001b[0;34m(lr_dim, Nlay, LSTM_DIM, key_dim, value_dim, atnheads, proj_dim, drop, embed_dim, extract)\u001b[0m\n\u001b[1;32m    286\u001b[0m         \u001b[0;31m#would be a bad to do inference on anyways so we skip that model and return a bad AUC.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    287\u001b[0m         \u001b[0;31m#try:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 288\u001b[0;31m         \u001b[0mhistory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_epoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalid_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalidation_steps\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcblist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    289\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    290\u001b[0m         \u001b[0;31m#except:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, max_queue_size, workers, use_multiprocessing, **kwargs)\u001b[0m\n\u001b[1;32m   1637\u001b[0m           \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1638\u001b[0m           \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1639\u001b[0;31m           validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1640\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1641\u001b[0m   def evaluate(self,\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/tensorflow/python/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, inputs, targets, sample_weights, batch_size, epochs, verbose, callbacks, val_inputs, val_targets, val_sample_weights, shuffle, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    237\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics_names\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_outs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m               \u001b[0mepoch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'val_'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0ml\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 239\u001b[0;31m     \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    240\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m       \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m    212\u001b[0m     \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlogs\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m       \u001b[0mcallback\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_epoch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    215\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/AI/noovle-open-source/Fit_Function.py\u001b[0m in \u001b[0;36mon_epoch_end\u001b[0;34m(self, epoch, logs)\u001b[0m\n\u001b[1;32m    189\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minterval\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m>\u001b[0m\u001b[0mwait_epochs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m                 \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX_val\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m                 \u001b[0mscore\u001b[0m\u001b[0;34m=\u001b[0m \u001b[0mEvalue_fun_simple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0my_val\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscore_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m                 \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/AI/noovle-open-source/Fit_Function.py\u001b[0m in \u001b[0;36mEvalue_fun_simple\u001b[0;34m(pred, label)\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0mpred_BOW_eval\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpred\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcla_to_keep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m         \u001b[0;31m#This takes forever\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m         \u001b[0mmacro_auc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mroc_auc_score\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_L_eval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpred_BOW_eval\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0maverage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mevaluation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \u001b[0;32mreturn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmacro_auc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/sklearn/metrics/ranking.py\u001b[0m in \u001b[0;36mroc_auc_score\u001b[0;34m(y_true, y_score, average, sample_weight)\u001b[0m\n\u001b[1;32m    275\u001b[0m     return _average_binary_score(\n\u001b[1;32m    276\u001b[0m         \u001b[0m_binary_roc_auc_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_score\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maverage\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 277\u001b[0;31m         sample_weight=sample_weight)\n\u001b[0m\u001b[1;32m    278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    279\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/pytorch/lib/python3.6/site-packages/sklearn/metrics/base.py\u001b[0m in \u001b[0;36m_average_binary_score\u001b[0;34m(binary_metric, y_true, y_score, average, sample_weight)\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0mscore\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_classes\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_classes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m         \u001b[0my_true_c\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_true\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnot_average_axis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m         \u001b[0my_score_c\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my_score\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnot_average_axis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         score[c] = binary_metric(y_true_c, y_score_c,\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Hyper_fit(# Number of Training steps\n",
    "          steps=50000\n",
    "    \n",
    "          #where to save the best model/name\n",
    "          ,path_best_model='best_model.keras'\n",
    "    \n",
    "          #folder for the Tensorboard Log files\n",
    "          ,TB_logs='logs'\n",
    "        \n",
    "          ,train_files=['tf_rec/MIMIC_train.tfrecords']\n",
    "          #the validation files\n",
    "    \n",
    "          ,validation_files='tf_rec/MIMIC_validation.tfrecords'\n",
    "    \n",
    "          #the files for the tokenizers\n",
    "          ,label_tokenizer_path='tokenizer/Label_Tokenizer.pkl'\n",
    "          ,text_tokenizer_path='tokenizer/Text_Tokenizer.pkl'\n",
    "    \n",
    "          #The numer of Hyperparameter rounds\n",
    "          ,rounds=100\n",
    "    \n",
    "          #max number of epochs we want to train\n",
    "          ,max_epoch=50\n",
    "    \n",
    "          #buffer size used for shuffling should be batch size for best results\n",
    "          ,buffer_size=50000\n",
    "    \n",
    "          #if we want to use Hyper opt or just get a single fit\n",
    "          ,single_fit=False\n",
    "    \n",
    "          #one of weighted, macro or micro for the validation AUC weighted is the most robust one\n",
    "          ,evaluation=\"macro\"\n",
    "    \n",
    "          #the default parameters for training, they will be the only parameters used for training\n",
    "          #lr_dim,Nlay,LSTM_DIM,key_dim,value_dim,atnheads,proj_dim,drop,embed_dim ,extract\n",
    "          ,default_parameters=[90,3,78,128,128,20,128,0.5,100,'lstm']\n",
    "    \n",
    "          #if we want to train a baseline, careful only the baseline will be trained, so only use with single fit. \n",
    "          ,baseline=False\n",
    "          #the intervall tells us when to calculate the evaluation ( given any method), with many output classes\n",
    "          #this can take a long time\n",
    "          ,interval=1\n",
    "         #Again because the caluclation of the metrics can take along time we can subsample the Classes we eval on, \n",
    "         #this saves zalot of time it has to be less than the nuber of classes \n",
    "          ,eval_size=2500\n",
    "         #If the model for inference will run on CPU or GPU (even if there is a liklihood that it runs on CPU put it to yes)\n",
    "         # A model saved on cpu can run on gpu but not other way round. \n",
    "    \n",
    "          ,save_cpu=True\n",
    "    \n",
    "          ,batch_size=128)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (pytorchenv)",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}