Train OpenMIIR TripletNet RAW simpleConv 1x1x1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "job_id = 'raw_openmiir_tripletnet_simpleconv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX TITAN X (CNMeM is disabled, cuDNN 5105)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "blocks default_seed: {'default': 1, 'type': <type 'int'>, 'value': 42}\n",
      "batch_size = 1000\n",
      "blocks_extensions_bokeh = False\n",
      "blocks_extensions_bokeh_channels = [['cost_train', 'cost_valid'], ['total_gradient_norm_train']]\n",
      "blocks_extensions_bokeh_plot_title = raw_openmiir_tripletnet_simpleconv\n",
      "blocks_extensions_printing = False\n",
      "blocks_extensions_progressbar = True\n",
      "blocks_extensions_train_monitoring_channels = []\n",
      "blocks_extensions_valid_monitoring_channels = []\n",
      "classification_target_source = targets\n",
      "classifier_batch_size = 120\n",
      "classifier_l1wdecay = 0.0\n",
      "classifier_max_epochs = 100\n",
      "classifier_pool_stride = 1\n",
      "classifier_pool_width = 1\n",
      "filter_width_freq = 1\n",
      "filter_width_time = 1\n",
      "group_attribute = within_subject_tuples_group\n",
      "l1wdecay = 0\n",
      "learning_rate = 0.01\n",
      "max_epochs = 10\n",
      "num_components = 1\n",
      "only_1_inner_fold = False\n",
      "pool_stride_freq = 1\n",
      "pool_stride_time = 1\n",
      "pool_width_freq = 1\n",
      "pool_width_time = 1\n",
      "pretrain_target_source = targets\n",
      "step_rule = Adam\n",
      "use_bias = False\n",
      "use_ext_dataset_for_validation = True\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"THEANO_FLAGS\"] = \"floatX=float32,device=gpu\"\n",
    "\n",
    "from common_settings import *\n",
    "hyper_params['blocks_extensions_bokeh_plot_title'] = job_id\n",
    "\n",
    "print_hyper_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from deepthought.experiments.encoding.datasets.openmiir import OpenMIIRNestedCVFoldGenerator\n",
    "from deepthought.experiments.encoding.experiment_templates.triplet_network import TripletNetworkExperiment\n",
    "from deepthought.experiments.encoding.pipelines.single_conv_filter import SingleConvFilterPipelineFactory\n",
    "from deepthought.experiments.encoding.classifiers.linear_svc import LinearSVCClassifierFactory\n",
    "from deepthought.experiments.encoding.classifiers.linear_svc import UntunedLinearSVCClassifierFactory\n",
    "from deepthought.experiments.encoding.classifiers.simple_nn import SimpleNNClassifierFactory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running job #raw_openmiir_tripletnet_simpleconv\n",
      "output path:  results/raw_openmiir_tripletnet_simpleconv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 0.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing outer fold {'train': ['P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P01']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 0.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 0.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 0.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 0.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 236.1804  237.1091  237.5259  242.8781  241.1833]\n",
      "fold_scores for c=0.0100: [ 0.3958  0.2708  0.1875  0.1562  0.2708] mean=0.2562\n",
      "fold_scores for c=0.0500: [ 0.3958  0.2708  0.1875  0.1667  0.2708] mean=0.2583\n",
      "fold_scores for c=0.1000: [ 0.3333  0.2292  0.2188  0.2188  0.2604] mean=0.2521\n",
      "fold_scores for c=0.0050: [ 0.3958  0.2708  0.1875  0.1562  0.2708] mean=0.2562\n",
      "fold_scores for c=0.0010: [ 0.3958  0.2708  0.1875  0.1562  0.2708] mean=0.2562\n",
      "fold_scores for c=0.0005: [ 0.3958  0.2708  0.1875  0.1562  0.2708] mean=0.2562\n",
      "fold_scores for c=0.0001: [ 0.3958  0.2708  0.1875  0.1562  0.2708] mean=0.2562\n",
      "fold_scores for c=0.5000: [ 0.3958  0.2604  0.2292  0.1771  0.3229] mean=0.2771\n",
      "fold_scores for c=1.0000: [ 0.3646  0.2812  0.2292  0.1771  0.2917] mean=0.2687\n",
      "fold_scores for c=2.0000: [ 0.3542  0.25    0.2396  0.1667  0.2188] mean=0.2458\n",
      "best c=0.5 with score 0.277083333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 1.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P01']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.17      0.20      0.18         5\n",
      "          1       0.14      0.20      0.17         5\n",
      "          2       0.25      0.40      0.31         5\n",
      "          3       0.67      0.40      0.50         5\n",
      "          4       0.25      0.40      0.31         5\n",
      "          5       0.25      0.20      0.22         5\n",
      "          6       0.60      0.60      0.60         5\n",
      "          7       0.50      0.40      0.44         5\n",
      "          8       0.00      0.00      0.00         5\n",
      "          9       0.00      0.00      0.00         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.40      0.40      0.40         5\n",
      "\n",
      "avg / total       0.27      0.27      0.26        60\n",
      "\n",
      "[[1 1 1 0 1 0 0 0 0 0 1 0]\n",
      " [1 1 0 0 1 1 0 0 0 0 0 1]\n",
      " [0 0 2 0 0 0 1 0 0 0 2 0]\n",
      " [0 1 0 2 0 1 0 1 0 0 0 0]\n",
      " [1 1 0 0 2 0 0 0 1 0 0 0]\n",
      " [1 2 0 1 0 1 0 0 0 0 0 0]\n",
      " [0 0 0 0 1 0 3 0 0 0 1 0]\n",
      " [1 0 0 0 0 1 0 2 0 1 0 0]\n",
      " [0 1 1 0 0 0 0 0 0 2 0 1]\n",
      " [0 0 1 0 1 0 1 1 0 0 0 1]\n",
      " [0 0 3 0 2 0 0 0 0 0 0 0]\n",
      " [1 0 0 0 0 0 0 0 1 1 0 2]]\n",
      "overall test accuracy so far: 0.266666666667\n",
      "processing outer fold {'train': ['P01', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P04']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 1.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 1.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 1.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 1.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 231.6534  234.0346  241.9051  243.5457  243.5142]\n",
      "fold_scores for c=0.0100: [ 0.2917  0.1875  0.1562  0.1979  0.3125] mean=0.2292\n",
      "fold_scores for c=0.0500: [ 0.3021  0.2396  0.1146  0.2083  0.3333] mean=0.2396\n",
      "fold_scores for c=0.1000: [ 0.3229  0.2292  0.2083  0.2396  0.3438] mean=0.2687\n",
      "fold_scores for c=0.0050: [ 0.2917  0.1875  0.1562  0.1979  0.3125] mean=0.2292\n",
      "fold_scores for c=0.0010: [ 0.2917  0.1875  0.1562  0.1979  0.3125] mean=0.2292\n",
      "fold_scores for c=0.0005: [ 0.2917  0.1875  0.1562  0.1979  0.3125] mean=0.2292\n",
      "fold_scores for c=0.0001: [ 0.2917  0.1875  0.1562  0.1979  0.3125] mean=0.2292\n",
      "fold_scores for c=0.5000: [ 0.3229  0.2292  0.1771  0.2292  0.3125] mean=0.2542\n",
      "fold_scores for c=1.0000: [ 0.3438  0.2396  0.1667  0.2292  0.3333] mean=0.2625\n",
      "fold_scores for c=2.0000: [ 0.3438  0.2396  0.1667  0.2292  0.3125] mean=0.2583\n",
      "best c=0.1 with score 0.26875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 2.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P04']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.67      0.40      0.50         5\n",
      "          1       0.00      0.00      0.00         5\n",
      "          2       0.38      0.60      0.46         5\n",
      "          3       0.22      0.40      0.29         5\n",
      "          4       0.50      0.60      0.55         5\n",
      "          5       0.17      0.20      0.18         5\n",
      "          6       0.25      0.20      0.22         5\n",
      "          7       0.33      0.40      0.36         5\n",
      "          8       0.50      0.20      0.29         5\n",
      "          9       0.17      0.20      0.18         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.75      0.60      0.67         5\n",
      "\n",
      "avg / total       0.33      0.32      0.31        60\n",
      "\n",
      "[[2 1 1 0 1 0 0 0 0 0 0 0]\n",
      " [0 0 0 2 0 3 0 0 0 0 0 0]\n",
      " [0 0 3 0 0 0 0 0 0 1 1 0]\n",
      " [0 1 1 2 0 0 0 1 0 0 0 0]\n",
      " [1 0 0 0 3 1 0 0 0 0 0 0]\n",
      " [0 0 0 2 0 1 0 0 0 1 0 1]\n",
      " [0 0 1 0 1 0 1 1 0 0 1 0]\n",
      " [0 0 0 2 0 0 0 2 1 0 0 0]\n",
      " [0 0 0 0 1 0 1 0 1 2 0 0]\n",
      " [0 1 0 0 0 0 1 2 0 1 0 0]\n",
      " [0 0 2 1 0 0 1 0 0 1 0 0]\n",
      " [0 1 0 0 0 1 0 0 0 0 0 3]]\n",
      "overall test accuracy so far: 0.291666666667\n",
      "processing outer fold {'train': ['P01', 'P04', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P06']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 2.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 2.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 2.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 2.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 230.7425  234.8105  238.6879  238.9396  244.8006]\n",
      "fold_scores for c=0.0100: [ 0.3125  0.2708  0.1875  0.2292  0.3333] mean=0.2667\n",
      "fold_scores for c=0.0500: [ 0.3229  0.2812  0.2604  0.3229  0.3438] mean=0.3063\n",
      "fold_scores for c=0.1000: [ 0.375   0.3229  0.1979  0.2604  0.3021] mean=0.2917\n",
      "fold_scores for c=0.0050: [ 0.3125  0.2708  0.1875  0.2292  0.3333] mean=0.2667\n",
      "fold_scores for c=0.0010: [ 0.3125  0.2708  0.1875  0.2292  0.3333] mean=0.2667\n",
      "fold_scores for c=0.0005: [ 0.3125  0.2708  0.1875  0.2292  0.3333] mean=0.2667\n",
      "fold_scores for c=0.0001: [ 0.3125  0.2708  0.1875  0.2292  0.3333] mean=0.2667\n",
      "fold_scores for c=0.5000: [ 0.2917  0.2083  0.2396  0.2292  0.3229] mean=0.2583\n",
      "fold_scores for c=1.0000: [ 0.2917  0.2083  0.2292  0.2292  0.3229] mean=0.2562\n",
      "fold_scores for c=2.0000: [ 0.2917  0.2083  0.2292  0.2292  0.3229] mean=0.2562\n",
      "best c=0.05 with score 0.30625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 3.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P06']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       1.00      0.20      0.33         5\n",
      "          1       0.17      0.20      0.18         5\n",
      "          2       0.12      0.20      0.15         5\n",
      "          3       0.43      0.60      0.50         5\n",
      "          4       0.00      0.00      0.00         5\n",
      "          5       0.00      0.00      0.00         5\n",
      "          6       0.33      0.20      0.25         5\n",
      "          7       0.00      0.00      0.00         5\n",
      "          8       0.33      0.40      0.36         5\n",
      "          9       0.12      0.20      0.15         5\n",
      "         10       0.20      0.20      0.20         5\n",
      "         11       0.40      0.40      0.40         5\n",
      "\n",
      "avg / total       0.26      0.22      0.21        60\n",
      "\n",
      "[[1 0 0 0 2 1 0 0 0 1 0 0]\n",
      " [0 1 1 0 0 0 1 0 1 1 0 0]\n",
      " [0 1 1 0 0 1 0 0 0 0 2 0]\n",
      " [0 0 1 3 0 0 0 0 0 0 1 0]\n",
      " [0 1 0 1 0 0 1 0 0 0 0 2]\n",
      " [0 0 0 1 1 0 0 1 2 0 0 0]\n",
      " [0 0 2 0 1 0 1 0 0 1 0 0]\n",
      " [0 0 0 1 1 0 0 0 0 2 1 0]\n",
      " [0 1 1 0 0 0 0 0 2 1 0 0]\n",
      " [0 1 0 0 1 0 0 1 0 1 0 1]\n",
      " [0 0 2 0 0 0 0 0 1 1 1 0]\n",
      " [0 1 0 1 1 0 0 0 0 0 0 2]]\n",
      "overall test accuracy so far: 0.266666666667\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P07']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 3.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 3.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 3.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 3.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 232.7457  228.0043  238.6713  238.8808  239.8131]\n",
      "fold_scores for c=0.0100: [ 0.3125  0.2292  0.1667  0.3646  0.3438] mean=0.2833\n",
      "fold_scores for c=0.0500: [ 0.3542  0.2604  0.2188  0.3021  0.3646] mean=0.3000\n",
      "fold_scores for c=0.1000: [ 0.3333  0.2292  0.1875  0.3021  0.375 ] mean=0.2854\n",
      "fold_scores for c=0.0050: [ 0.3125  0.2188  0.1667  0.3646  0.3438] mean=0.2812\n",
      "fold_scores for c=0.0010: [ 0.3125  0.2188  0.1667  0.3646  0.3438] mean=0.2812\n",
      "fold_scores for c=0.0005: [ 0.3125  0.2188  0.1667  0.3646  0.3438] mean=0.2812\n",
      "fold_scores for c=0.0001: [ 0.3125  0.2188  0.1667  0.3646  0.3438] mean=0.2812\n",
      "fold_scores for c=0.5000: [ 0.3646  0.2396  0.2188  0.3229  0.375 ] mean=0.3042\n",
      "fold_scores for c=1.0000: [ 0.3646  0.2396  0.2188  0.3229  0.3854] mean=0.3063\n",
      "fold_scores for c=2.0000: [ 0.3646  0.2396  0.2188  0.3229  0.3854] mean=0.3063\n",
      "best c=1 with score 0.30625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 4.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P07']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.38      0.60      0.46         5\n",
      "          1       0.17      0.20      0.18         5\n",
      "          2       0.40      0.40      0.40         5\n",
      "          3       0.14      0.20      0.17         5\n",
      "          4       0.25      0.20      0.22         5\n",
      "          5       0.67      0.40      0.50         5\n",
      "          6       0.00      0.00      0.00         5\n",
      "          7       0.33      0.40      0.36         5\n",
      "          8       0.25      0.20      0.22         5\n",
      "          9       0.00      0.00      0.00         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.25      0.20      0.22         5\n",
      "\n",
      "avg / total       0.24      0.23      0.23        60\n",
      "\n",
      "[[3 0 0 0 0 0 0 1 0 1 0 0]\n",
      " [0 1 0 1 0 0 1 1 0 1 0 0]\n",
      " [0 0 2 1 0 1 0 0 1 0 0 0]\n",
      " [0 1 0 1 1 0 0 1 0 0 1 0]\n",
      " [0 1 0 0 1 0 1 0 0 1 0 1]\n",
      " [2 0 0 0 0 2 0 1 0 0 0 0]\n",
      " [1 0 0 0 1 0 0 0 1 1 1 0]\n",
      " [0 0 0 1 1 0 1 2 0 0 0 0]\n",
      " [1 0 1 2 0 0 0 0 1 0 0 0]\n",
      " [1 1 1 0 0 0 0 0 0 0 1 1]\n",
      " [0 1 1 0 0 0 0 0 1 1 0 1]\n",
      " [0 1 0 1 0 0 1 0 0 1 0 1]]\n",
      "overall test accuracy so far: 0.258333333333\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P09']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 4.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 4.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 4.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 4.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 237.7071  235.0535  238.4974  239.3994  237.9163]\n",
      "fold_scores for c=0.0100: [ 0.3021  0.2083  0.1562  0.2812  0.2708] mean=0.2437\n",
      "fold_scores for c=0.0500: [ 0.3229  0.1667  0.1875  0.2917  0.375 ] mean=0.2687\n",
      "fold_scores for c=0.1000: [ 0.3229  0.1875  0.2083  0.2604  0.3125] mean=0.2583\n",
      "fold_scores for c=0.0050: [ 0.3021  0.2083  0.1562  0.2812  0.2708] mean=0.2437\n",
      "fold_scores for c=0.0010: [ 0.3021  0.2083  0.1562  0.2812  0.2708] mean=0.2437\n",
      "fold_scores for c=0.0005: [ 0.3021  0.2083  0.1562  0.2812  0.2708] mean=0.2437\n",
      "fold_scores for c=0.0001: [ 0.3021  0.2083  0.1562  0.2812  0.2708] mean=0.2437\n",
      "fold_scores for c=0.5000: [ 0.3646  0.1875  0.1979  0.25    0.3125] mean=0.2625\n",
      "fold_scores for c=1.0000: [ 0.3646  0.1875  0.1979  0.25    0.3125] mean=0.2625\n",
      "fold_scores for c=2.0000: [ 0.3646  0.1875  0.1979  0.25    0.3125] mean=0.2625\n",
      "best c=0.05 with score 0.26875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stober/.local/lib/python2.7/site-packages/sklearn/metrics/classification.py:958: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "INFO:deepthought.experiments:processing fold 5.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P07', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P09']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.25      0.20      0.22         5\n",
      "          1       0.40      0.80      0.53         5\n",
      "          2       0.67      0.40      0.50         5\n",
      "          3       0.50      0.20      0.29         5\n",
      "          4       0.17      0.20      0.18         5\n",
      "          5       1.00      0.60      0.75         5\n",
      "          6       0.44      0.80      0.57         5\n",
      "          7       0.43      0.60      0.50         5\n",
      "          8       0.50      0.60      0.55         5\n",
      "          9       0.50      0.20      0.29         5\n",
      "         10       0.38      0.60      0.46         5\n",
      "         11       0.00      0.00      0.00         5\n",
      "\n",
      "avg / total       0.44      0.43      0.40        60\n",
      "\n",
      "[[1 1 0 0 2 0 0 1 0 0 0 0]\n",
      " [0 4 0 0 0 0 1 0 0 0 0 0]\n",
      " [0 1 2 0 0 0 0 0 0 0 2 0]\n",
      " [1 1 0 1 0 0 0 2 0 0 0 0]\n",
      " [2 0 0 0 1 0 1 0 1 0 0 0]\n",
      " [0 1 0 0 1 3 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 4 0 1 0 0 0]\n",
      " [0 0 0 1 0 0 0 3 0 1 0 0]\n",
      " [0 2 0 0 0 0 0 0 3 0 0 0]\n",
      " [0 0 0 0 0 0 2 0 0 1 2 0]\n",
      " [0 0 1 0 0 0 1 0 0 0 3 0]\n",
      " [0 0 0 0 2 0 0 1 1 0 1 0]]\n",
      "overall test accuracy so far: 0.293333333333\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P12', 'P13', 'P14'], 'valid': ['P11']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 5.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 5.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 5.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 5.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 233.4903  231.3354  237.0861  238.7207  243.66  ]\n",
      "fold_scores for c=0.0100: [ 0.2812  0.2396  0.2396  0.2917  0.3021] mean=0.2708\n",
      "fold_scores for c=0.0500: [ 0.375   0.2917  0.25    0.3438  0.3333] mean=0.3187\n",
      "fold_scores for c=0.1000: [ 0.375   0.2292  0.2292  0.3021  0.3646] mean=0.3000\n",
      "fold_scores for c=0.0050: [ 0.2812  0.2396  0.2396  0.2917  0.3021] mean=0.2708\n",
      "fold_scores for c=0.0010: [ 0.2812  0.2396  0.2396  0.2917  0.3021] mean=0.2708\n",
      "fold_scores for c=0.0005: [ 0.2812  0.2396  0.2396  0.2917  0.3021] mean=0.2708\n",
      "fold_scores for c=0.0001: [ 0.2812  0.2396  0.2396  0.2917  0.3021] mean=0.2708\n",
      "fold_scores for c=0.5000: [ 0.3333  0.2396  0.25    0.3021  0.3021] mean=0.2854\n",
      "fold_scores for c=1.0000: [ 0.3333  0.2396  0.25    0.3021  0.3021] mean=0.2854\n",
      "fold_scores for c=2.0000: [ 0.3333  0.2396  0.25    0.3021  0.3021] mean=0.2854\n",
      "best c=0.05 with score 0.31875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 6.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P12', 'P13', 'P14'], 'valid': ['P11']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.12      0.20      0.15         5\n",
      "          1       0.00      0.00      0.00         5\n",
      "          2       0.11      0.20      0.14         5\n",
      "          3       0.00      0.00      0.00         5\n",
      "          4       0.00      0.00      0.00         5\n",
      "          5       0.25      0.40      0.31         5\n",
      "          6       0.50      0.20      0.29         5\n",
      "          7       0.00      0.00      0.00         5\n",
      "          8       0.00      0.00      0.00         5\n",
      "          9       0.17      0.20      0.18         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.50      0.60      0.55         5\n",
      "\n",
      "avg / total       0.14      0.15      0.13        60\n",
      "\n",
      "[[1 0 2 0 1 1 0 0 0 0 0 0]\n",
      " [2 0 0 0 1 1 0 0 0 1 0 0]\n",
      " [1 2 1 0 0 1 0 0 0 0 0 0]\n",
      " [0 0 2 0 0 1 0 1 0 0 1 0]\n",
      " [3 0 0 0 0 0 1 0 0 0 0 1]\n",
      " [0 0 0 2 0 2 0 0 0 0 0 1]\n",
      " [0 0 0 0 1 0 1 0 0 1 1 1]\n",
      " [0 0 0 2 0 0 0 0 1 1 1 0]\n",
      " [0 1 1 0 0 0 0 0 0 1 2 0]\n",
      " [1 0 1 0 1 1 0 0 0 1 0 0]\n",
      " [0 0 2 1 0 1 0 0 0 1 0 0]\n",
      " [0 1 0 0 0 0 0 1 0 0 0 3]]\n",
      "overall test accuracy so far: 0.269444444444\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P13', 'P14'], 'valid': ['P12']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 6.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 6.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 6.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 6.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 231.449   234.5654  239.7923  241.1266  241.0365]\n",
      "fold_scores for c=0.0100: [ 0.3229  0.2292  0.1875  0.3125  0.2812] mean=0.2667\n",
      "fold_scores for c=0.0500: [ 0.3438  0.2396  0.25    0.2812  0.375 ] mean=0.2979\n",
      "fold_scores for c=0.1000: [ 0.3854  0.25    0.2188  0.3021  0.375 ] mean=0.3063\n",
      "fold_scores for c=0.0050: [ 0.3229  0.2292  0.1875  0.3125  0.2812] mean=0.2667\n",
      "fold_scores for c=0.0010: [ 0.3229  0.2292  0.1875  0.3125  0.2812] mean=0.2667\n",
      "fold_scores for c=0.0005: [ 0.3229  0.2292  0.1875  0.3125  0.2812] mean=0.2667\n",
      "fold_scores for c=0.0001: [ 0.3229  0.2292  0.1875  0.3125  0.2812] mean=0.2667\n",
      "fold_scores for c=0.5000: [ 0.3333  0.1979  0.2292  0.25    0.3438] mean=0.2708\n",
      "fold_scores for c=1.0000: [ 0.3333  0.1979  0.2292  0.25    0.3438] mean=0.2708\n",
      "fold_scores for c=2.0000: [ 0.3333  0.1979  0.2292  0.25    0.3438] mean=0.2708\n",
      "best c=0.1 with score 0.30625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 7.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P13', 'P14'], 'valid': ['P12']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00         5\n",
      "          1       0.38      0.60      0.46         5\n",
      "          2       0.17      0.20      0.18         5\n",
      "          3       0.00      0.00      0.00         5\n",
      "          4       0.40      0.40      0.40         5\n",
      "          5       0.40      0.40      0.40         5\n",
      "          6       0.33      0.40      0.36         5\n",
      "          7       0.00      0.00      0.00         5\n",
      "          8       0.00      0.00      0.00         5\n",
      "          9       0.00      0.00      0.00         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.25      0.20      0.22         5\n",
      "\n",
      "avg / total       0.16      0.18      0.17        60\n",
      "\n",
      "[[0 0 0 1 2 0 0 0 1 0 0 1]\n",
      " [0 3 0 1 0 0 0 0 0 0 1 0]\n",
      " [0 0 1 1 0 0 0 0 1 1 1 0]\n",
      " [1 1 1 0 0 0 0 0 1 0 1 0]\n",
      " [1 0 0 0 2 2 0 0 0 0 0 0]\n",
      " [0 1 0 0 0 2 0 1 0 0 0 1]\n",
      " [0 0 2 0 0 0 2 0 0 1 0 0]\n",
      " [0 0 0 1 0 1 1 0 0 0 1 1]\n",
      " [0 2 1 1 1 0 0 0 0 0 0 0]\n",
      " [2 0 1 1 0 0 0 0 0 0 1 0]\n",
      " [1 1 0 0 0 0 2 0 1 0 0 0]\n",
      " [0 0 0 1 0 0 1 0 1 0 1 1]]\n",
      "overall test accuracy so far: 0.257142857143\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P14'], 'valid': ['P13']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 7.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 7.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 7.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 7.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 229.8308  230.9349  236.1713  236.4818  241.9727]\n",
      "fold_scores for c=0.0100: [ 0.2292  0.25    0.2083  0.3021  0.3125] mean=0.2604\n",
      "fold_scores for c=0.0500: [ 0.375   0.2292  0.2188  0.375   0.3646] mean=0.3125\n",
      "fold_scores for c=0.1000: [ 0.3542  0.2917  0.2396  0.3438  0.3229] mean=0.3104\n",
      "fold_scores for c=0.0050: [ 0.2292  0.25    0.2083  0.3021  0.3125] mean=0.2604\n",
      "fold_scores for c=0.0010: [ 0.2292  0.25    0.2083  0.3021  0.3125] mean=0.2604\n",
      "fold_scores for c=0.0005: [ 0.2292  0.25    0.2083  0.3021  0.3125] mean=0.2604\n",
      "fold_scores for c=0.0001: [ 0.2292  0.25    0.2083  0.3021  0.3125] mean=0.2604\n",
      "fold_scores for c=0.5000: [ 0.375   0.2188  0.2708  0.2917  0.3229] mean=0.2958\n",
      "fold_scores for c=1.0000: [ 0.375   0.2188  0.2708  0.2917  0.3229] mean=0.2958\n",
      "fold_scores for c=2.0000: [ 0.375   0.2188  0.2708  0.2917  0.3229] mean=0.2958\n",
      "best c=0.05 with score 0.3125\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:deepthought.experiments:processing fold 8.0: {'train': [1, 2, 3, 4], 'valid': [0]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P14'], 'valid': ['P13']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.60      0.60      0.60         5\n",
      "          1       0.22      0.40      0.29         5\n",
      "          2       0.50      0.20      0.29         5\n",
      "          3       0.40      0.40      0.40         5\n",
      "          4       0.00      0.00      0.00         5\n",
      "          5       0.25      0.20      0.22         5\n",
      "          6       0.22      0.40      0.29         5\n",
      "          7       0.00      0.00      0.00         5\n",
      "          8       0.40      0.40      0.40         5\n",
      "          9       0.33      0.60      0.43         5\n",
      "         10       0.00      0.00      0.00         5\n",
      "         11       0.33      0.20      0.25         5\n",
      "\n",
      "avg / total       0.27      0.28      0.26        60\n",
      "\n",
      "[[3 0 0 0 1 0 0 1 0 0 0 0]\n",
      " [0 2 0 0 2 0 0 0 0 0 1 0]\n",
      " [0 0 1 0 0 1 2 0 0 1 0 0]\n",
      " [0 1 0 2 0 0 0 0 0 2 0 0]\n",
      " [1 1 0 0 0 0 1 0 0 0 0 2]\n",
      " [0 1 0 1 1 1 0 0 1 0 0 0]\n",
      " [0 0 0 0 0 0 2 1 0 1 1 0]\n",
      " [1 1 1 0 0 0 1 0 1 0 0 0]\n",
      " [0 1 0 0 0 1 0 0 2 1 0 0]\n",
      " [0 0 0 0 0 0 0 0 1 3 1 0]\n",
      " [0 2 0 1 0 0 2 0 0 0 0 0]\n",
      " [0 0 0 1 0 1 1 0 0 1 0 1]]\n",
      "overall test accuracy so far: 0.260416666667\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13'], 'valid': ['P14']}\n",
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 8.1: {'train': [0, 2, 3, 4], 'valid': [1]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 8.2: {'train': [0, 1, 3, 4], 'valid': [2]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 8.3: {'train': [0, 1, 2, 4], 'valid': [3]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "INFO:deepthought.experiments:processing fold 8.4: {'train': [0, 1, 2, 3], 'valid': [4]}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#train: 50688 #valid: 21120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 1, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 2, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 3, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 4, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 5, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 6, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 7, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 8, step 51 |                                       | Elapsed Time: 0:00:06\n",
      "Epoch 9, step 51 |                                       | Elapsed Time: 0:00:06\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold errors: [ 232.3251  230.0878  241.0646  238.8288  239.017 ]\n",
      "fold_scores for c=0.0100: [ 0.3125  0.2917  0.2083  0.2917  0.3021] mean=0.2812\n",
      "fold_scores for c=0.0500: [ 0.3229  0.2917  0.2292  0.2812  0.3438] mean=0.2938\n",
      "fold_scores for c=0.1000: [ 0.3125  0.2708  0.2188  0.25    0.3229] mean=0.2750\n",
      "fold_scores for c=0.0050: [ 0.3125  0.2917  0.2083  0.2917  0.3021] mean=0.2812\n",
      "fold_scores for c=0.0010: [ 0.3125  0.2917  0.2083  0.2917  0.3021] mean=0.2812\n",
      "fold_scores for c=0.0005: [ 0.3125  0.2917  0.2083  0.2917  0.3021] mean=0.2812\n",
      "fold_scores for c=0.0001: [ 0.3125  0.2917  0.2083  0.2917  0.3021] mean=0.2812\n",
      "fold_scores for c=0.5000: [ 0.2708  0.2188  0.2188  0.25    0.3125] mean=0.2542\n",
      "fold_scores for c=1.0000: [ 0.2708  0.2188  0.2188  0.25    0.3125] mean=0.2542\n",
      "fold_scores for c=2.0000: [ 0.2708  0.2188  0.2188  0.25    0.3125] mean=0.2542\n",
      "best c=0.05 with score 0.29375\n",
      "linear_svc results for fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13'], 'valid': ['P14']}\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.00      0.00      0.00         5\n",
      "          1       0.25      0.60      0.35         5\n",
      "          2       0.50      0.40      0.44         5\n",
      "          3       0.00      0.00      0.00         5\n",
      "          4       0.25      0.20      0.22         5\n",
      "          5       0.20      0.20      0.20         5\n",
      "          6       0.00      0.00      0.00         5\n",
      "          7       0.25      0.20      0.22         5\n",
      "          8       0.00      0.00      0.00         5\n",
      "          9       0.30      0.60      0.40         5\n",
      "         10       0.50      0.20      0.29         5\n",
      "         11       0.25      0.20      0.22         5\n",
      "\n",
      "avg / total       0.21      0.22      0.20        60\n",
      "\n",
      "[[0 1 1 0 1 1 1 0 0 0 0 0]\n",
      " [0 3 0 0 0 0 0 2 0 0 0 0]\n",
      " [0 0 2 0 2 0 1 0 0 0 0 0]\n",
      " [1 0 0 0 0 1 0 1 2 0 0 0]\n",
      " [1 1 0 0 1 0 0 0 0 2 0 0]\n",
      " [0 3 0 0 0 1 0 0 0 1 0 0]\n",
      " [0 1 1 0 0 0 0 0 1 2 0 0]\n",
      " [2 1 0 0 0 0 1 1 0 0 0 0]\n",
      " [0 0 0 0 0 2 0 0 0 0 1 2]\n",
      " [0 1 0 0 0 0 0 0 0 3 0 1]\n",
      " [0 1 0 0 0 0 1 0 1 1 1 0]\n",
      " [0 0 0 1 0 0 0 0 2 1 0 1]]\n",
      "overall test accuracy so far: 0.255555555556\n",
      "all folds completed\n",
      "\n",
      "SUMMARY for classifier linear_svc\n",
      "\n",
      "fold scores:  [ 0.2667  0.3167  0.2167  0.2333  0.4333  0.15    0.1833  0.2833  0.2167]\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.27      0.27      0.27        45\n",
      "        1.0       0.23      0.33      0.27        45\n",
      "        2.0       0.28      0.33      0.31        45\n",
      "        3.0       0.24      0.24      0.24        45\n",
      "        4.0       0.21      0.22      0.22        45\n",
      "        5.0       0.33      0.29      0.31        45\n",
      "        6.0       0.30      0.31      0.31        45\n",
      "        7.0       0.29      0.22      0.25        45\n",
      "        8.0       0.24      0.20      0.22        45\n",
      "        9.0       0.19      0.22      0.20        45\n",
      "       10.0       0.13      0.11      0.12        45\n",
      "       11.0       0.40      0.31      0.35        45\n",
      "\n",
      "avg / total       0.26      0.26      0.26       540\n",
      "\n",
      "[[12  4  5  1 11  3  1  3  1  2  1  1]\n",
      " [ 3 15  1  4  4  5  3  3  1  3  2  1]\n",
      " [ 1  4 15  2  2  4  4  0  2  3  8  0]\n",
      " [ 3  6  5 11  1  3  0  7  3  2  4  0]\n",
      " [10  5  0  1 10  3  5  0  2  3  0  6]\n",
      " [ 3  8  0  7  3 13  0  3  3  2  0  3]\n",
      " [ 1  1  6  0  5  0 14  2  3  7  5  1]\n",
      " [ 4  2  1  8  2  2  4 10  3  5  3  1]\n",
      " [ 1  8  5  3  2  3  1  0  9  7  3  3]\n",
      " [ 4  4  4  1  3  1  4  4  1 10  5  4]\n",
      " [ 1  5 11  3  2  1  7  0  4  5  5  1]\n",
      " [ 1  4  0  5  3  2  3  2  5  4  2 14]]\n",
      "\n",
      "train accuracy: 0.874305555556\n",
      "test accuracy : 0.255555555556\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.74444444444444446]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp = TripletNetworkExperiment(job_id, hdf5name, \n",
    "                               OpenMIIRNestedCVFoldGenerator(),\n",
    "                               SingleConvFilterPipelineFactory(),\n",
    "                               hyper_params=hyper_params, default_params=None,\n",
    "                               base_selectors=base_selectors)\n",
    "\n",
    "# exp.run(classifiers=(('linear_svc_untuned', UntunedLinearSVCClassifierFactory()),), verbose=True)\n",
    "exp.run(classifiers=(('linear_svc', LinearSVCClassifierFactory()),), verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "running job #raw_openmiir_tripletnet_simpleconv\n",
      "output path:  results/raw_openmiir_tripletnet_simpleconv\n",
      "processing outer fold {'train': ['P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P01']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_0.pklz\n",
      "simple NN fold classification errors: [ 0.7188  0.7812  0.7292  0.7604  0.6979]\n",
      "processing outer fold {'train': ['P01', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P04']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_1.pklz\n",
      "simple NN fold classification errors: [ 0.7708  0.8021  0.7812  0.75    0.7083]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P07', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P06']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_2.pklz\n",
      "simple NN fold classification errors: [ 0.6667  0.7083  0.7292  0.7083  0.6979]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P09', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P07']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_3.pklz\n",
      "simple NN fold classification errors: [ 0.625   0.8021  0.7083  0.7188  0.6667]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P11', 'P12', 'P13', 'P14'], 'valid': ['P09']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_4.pklz\n",
      "simple NN fold classification errors: [ 0.7188  0.8125  0.7917  0.7083  0.75  ]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P12', 'P13', 'P14'], 'valid': ['P11']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_5.pklz\n",
      "simple NN fold classification errors: [ 0.6875  0.7708  0.7188  0.7083  0.6771]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P13', 'P14'], 'valid': ['P12']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_6.pklz\n",
      "simple NN fold classification errors: [ 0.6771  0.6979  0.7396  0.7292  0.6771]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P14'], 'valid': ['P13']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_7.pklz\n",
      "simple NN fold classification errors: [ 0.6875  0.7604  0.7396  0.6771  0.6562]\n",
      "processing outer fold {'train': ['P01', 'P04', 'P06', 'P07', 'P09', 'P11', 'P12', 'P13'], 'valid': ['P14']}\n",
      "loaded trained fold network parameters from results/raw_openmiir_tripletnet_simpleconv/fold_params_8.pklz\n",
      "simple NN fold classification errors: [ 0.6875  0.7083  0.7812  0.7188  0.6771]\n",
      "all folds completed\n",
      "\n",
      "SUMMARY for classifier simple_nn\n",
      "\n",
      "fold scores:  [ 0.2     0.2167  0.25    0.1667  0.4     0.1833  0.1667  0.2167  0.2   ]\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.27      0.36      0.30        45\n",
      "        1.0       0.19      0.18      0.18        45\n",
      "        2.0       0.23      0.29      0.25        45\n",
      "        3.0       0.22      0.16      0.18        45\n",
      "        4.0       0.14      0.13      0.14        45\n",
      "        5.0       0.26      0.22      0.24        45\n",
      "        6.0       0.24      0.29      0.26        45\n",
      "        7.0       0.19      0.20      0.20        45\n",
      "        8.0       0.23      0.22      0.22        45\n",
      "        9.0       0.20      0.13      0.16        45\n",
      "       10.0       0.08      0.07      0.07        45\n",
      "       11.0       0.35      0.42      0.38        45\n",
      "\n",
      "avg / total       0.22      0.22      0.22       540\n",
      "\n",
      "[[16  2  5  3  6  3  1  4  0  1  1  3]\n",
      " [ 5  8  1  3  4  5  3  4  4  2  3  3]\n",
      " [ 3  1 13  2  3  2  4  4  2  1  8  2]\n",
      " [ 3  4  6  7  3  3  3  6  4  1  4  1]\n",
      " [15  2  1  1  6  1  9  1  2  1  0  6]\n",
      " [ 4  6  1  3  4 10  3  4  5  1  1  3]\n",
      " [ 1  2  8  0  5  1 13  1  2  3  5  4]\n",
      " [ 3  2  3  8  2  4  5  9  3  3  2  1]\n",
      " [ 1  7  5  1  2  3  0  2 10  5  3  6]\n",
      " [ 5  2  6  1  3  2  3  5  2  6  6  4]\n",
      " [ 2  5  7  3  3  2  6  3  3  5  3  3]\n",
      " [ 2  1  1  0  2  2  4  4  7  1  2 19]]\n",
      "\n",
      "train accuracy: 0.685416666667\n",
      "test accuracy : 0.222222222222\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.77777777777777779]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.run(classifiers=(('simple_nn', SimpleNNClassifierFactory()),))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}