scc413_cw1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"SCC413 Coursework1.ipynb\n",
    "\n",
    "Automatically generated by Colaboratory.\n",
    "\n",
    "Original file is located at\n",
    "    https://colab.research.google.com/drive/1v7CoumwRlu7_Ose6HMNrlDpr2oCqtn3q\n",
    "\n",
    "#SCC413 Coursework 1\n",
    "\n",
    "### Imports\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import operator\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import tensorflow.keras as keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.interpolate import interp1d\n",
    "from scipy.io import loadmat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "\"\"\"### Test on ECG Data\n",
    "\n",
    "The dataset you will use is based on one from [timeseriesclassification.com](http://www.timeseriesclassification.com/description.php?Dataset=ECG5000).\n",
    "\n",
    "Try to design and train your MLP to classify normal and abnormal ECG samples.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"# Model Object Definitions\n",
    "\n",
    "## MLP Model\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier_MLP:\n",
    "\n",
    "\tdef __init__(self, output_directory, input_shape, nb_classes, verbose=False,build=True):\n",
    "\t\tself.output_directory = output_directory\n",
    "\t\tif build == True:\n",
    "\t\t\tself.model = self.build_model(input_shape, nb_classes)\n",
    "\t\t\tif(verbose==True):\n",
    "\t\t\t\tself.model.summary()\n",
    "\t\t\tself.verbose = verbose\n",
    "\t\t\tself.model.save_weights(self.output_directory + 'model_init.hdf5')\n",
    "\t\treturn\n",
    "\n",
    "\tdef build_model(self, input_shape, nb_classes):\n",
    "\t\tinput_layer = keras.layers.Input(input_shape)\n",
    "\n",
    "\t\t# flatten/reshape because when multivariate all should be on the same axis \n",
    "\t\tinput_layer_flattened = keras.layers.Flatten()(input_layer)\n",
    "\t\t\n",
    "\t\tlayer_1 = keras.layers.Dropout(0.1)(input_layer_flattened)\n",
    "\t\tlayer_1 = keras.layers.Dense(500, activation='relu')(layer_1)\n",
    "\n",
    "\t\tlayer_2 = keras.layers.Dropout(0.2)(layer_1)\n",
    "\t\tlayer_2 = keras.layers.Dense(500, activation='relu')(layer_2)\n",
    "\n",
    "\t\tlayer_3 = keras.layers.Dropout(0.2)(layer_2)\n",
    "\t\tlayer_3 = keras.layers.Dense(500, activation='relu')(layer_3)\n",
    "\n",
    "\t\toutput_layer = keras.layers.Dropout(0.3)(layer_3)\n",
    "\t\toutput_layer = keras.layers.Dense(nb_classes, activation='softmax')(output_layer)\n",
    "\n",
    "\t\tmodel = keras.models.Model(inputs=input_layer, outputs=output_layer)\n",
    "\n",
    "\t\tmodel.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adadelta(),\n",
    "\t\t\tmetrics=['accuracy'])\n",
    "\n",
    "\t\treduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=200, min_lr=0.1)\n",
    "\n",
    "\t\tfile_path = self.output_directory+'best_model.hdf5' \n",
    "\n",
    "\t\tmodel_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', \n",
    "\t\t\tsave_best_only=True)\n",
    "\n",
    "\t\tself.callbacks = [reduce_lr,model_checkpoint]\n",
    "\n",
    "\t\treturn model\n",
    "\n",
    "\tdef fit(self, x_train, y_train, x_val, y_val,y_true):\n",
    "\t\tif not tf.test.is_gpu_available:\n",
    "\t\t\tprint('error')\n",
    "\t\t\texit()\n",
    "\t\t# x_val and y_val are only used to monitor the test loss and NOT for training  \n",
    "\t\tbatch_size = 16\n",
    "\t\tnb_epochs = 500\n",
    "\n",
    "\t\tmini_batch_size = int(min(x_train.shape[0]/10, batch_size))\n",
    "\n",
    "\t\tstart_time = time.time() \n",
    "\n",
    "\t\thist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,\n",
    "\t\t\tverbose=self.verbose, validation_data=(x_val,y_val), callbacks=self.callbacks)\n",
    "\t\t\n",
    "\t\tduration = time.time() - start_time\n",
    "\n",
    "\t\tself.model.save(self.output_directory + 'last_model.hdf5')\n",
    "\n",
    "\t\tmodel = keras.models.load_model(self.output_directory+'best_model.hdf5')\n",
    "\n",
    "\t\ty_pred = model.predict(x_val)\n",
    "\n",
    "\t\t# convert the predicted from binary to integer \n",
    "\t\ty_pred = np.argmax(y_pred , axis=1)\n",
    "\n",
    "\t\tsave_logs(self.output_directory, hist, y_pred, y_true, duration)\n",
    "\n",
    "\t\tkeras.backend.clear_session()\n",
    "\n",
    "\tdef predict(self, x_test, y_true,x_train,y_train,y_test,return_df_metrics = True):\n",
    "\t\tmodel_path = self.output_directory + 'best_model.hdf5'\n",
    "\t\tmodel = keras.models.load_model(model_path)\n",
    "\t\ty_pred = model.predict(x_test)\n",
    "\t\tif return_df_metrics:\n",
    "\t\t\ty_pred = np.argmax(y_pred, axis=1)\n",
    "\t\t\tdf_metrics = calculate_metrics(y_true, y_pred, 0.0)\n",
    "\t\t\treturn df_metrics\n",
    "\t\telse:\n",
    "\t\t\treturn y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"## RESNET Model\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "class Classifier_RESNET:\n",
    "\n",
    "    def __init__(self, output_directory, input_shape, nb_classes, verbose=False, build=True, load_weights=False):\n",
    "        self.output_directory = output_directory\n",
    "        if build == True:\n",
    "            self.model = self.build_model(input_shape, nb_classes)\n",
    "            if (verbose == True):\n",
    "                self.model.summary()\n",
    "            self.verbose = verbose\n",
    "            if load_weights == True:\n",
    "                self.model.load_weights(self.output_directory\n",
    "                                        .replace('resnet_augment', 'resnet')\n",
    "                                        .replace('TSC_itr_augment_x_10', 'TSC_itr_10')\n",
    "                                        + '/model_init.hdf5')\n",
    "            else:\n",
    "                self.model.save_weights(self.output_directory + 'model_init.hdf5')\n",
    "        return\n",
    "\n",
    "    def build_model(self, input_shape, nb_classes):\n",
    "        n_feature_maps = 64\n",
    "\n",
    "        input_layer = keras.layers.Input(input_shape)\n",
    "\n",
    "        # BLOCK 1\n",
    "\n",
    "        conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer)\n",
    "        conv_x = keras.layers.BatchNormalization()(conv_x)\n",
    "        conv_x = keras.layers.Activation('relu')(conv_x)\n",
    "\n",
    "        conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x)\n",
    "        conv_y = keras.layers.BatchNormalization()(conv_y)\n",
    "        conv_y = keras.layers.Activation('relu')(conv_y)\n",
    "\n",
    "        conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y)\n",
    "        conv_z = keras.layers.BatchNormalization()(conv_z)\n",
    "\n",
    "        # expand channels for the sum\n",
    "        shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer)\n",
    "        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)\n",
    "\n",
    "        output_block_1 = keras.layers.add([shortcut_y, conv_z])\n",
    "        output_block_1 = keras.layers.Activation('relu')(output_block_1)\n",
    "\n",
    "        # BLOCK 2\n",
    "\n",
    "        conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1)\n",
    "        conv_x = keras.layers.BatchNormalization()(conv_x)\n",
    "        conv_x = keras.layers.Activation('relu')(conv_x)\n",
    "\n",
    "        conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)\n",
    "        conv_y = keras.layers.BatchNormalization()(conv_y)\n",
    "        conv_y = keras.layers.Activation('relu')(conv_y)\n",
    "\n",
    "        conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)\n",
    "        conv_z = keras.layers.BatchNormalization()(conv_z)\n",
    "\n",
    "        # expand channels for the sum\n",
    "        shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1)\n",
    "        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)\n",
    "\n",
    "        output_block_2 = keras.layers.add([shortcut_y, conv_z])\n",
    "        output_block_2 = keras.layers.Activation('relu')(output_block_2)\n",
    "\n",
    "        # BLOCK 3\n",
    "\n",
    "        conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2)\n",
    "        conv_x = keras.layers.BatchNormalization()(conv_x)\n",
    "        conv_x = keras.layers.Activation('relu')(conv_x)\n",
    "\n",
    "        conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)\n",
    "        conv_y = keras.layers.BatchNormalization()(conv_y)\n",
    "        conv_y = keras.layers.Activation('relu')(conv_y)\n",
    "\n",
    "        conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)\n",
    "        conv_z = keras.layers.BatchNormalization()(conv_z)\n",
    "\n",
    "        # no need to expand channels because they are equal\n",
    "        shortcut_y = keras.layers.BatchNormalization()(output_block_2)\n",
    "\n",
    "        output_block_3 = keras.layers.add([shortcut_y, conv_z])\n",
    "        output_block_3 = keras.layers.Activation('relu')(output_block_3)\n",
    "\n",
    "        # FINAL\n",
    "\n",
    "        gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)\n",
    "\n",
    "        output_layer = keras.layers.Dense(nb_classes, activation='softmax')(gap_layer)\n",
    "\n",
    "        model = keras.models.Model(inputs=input_layer, outputs=output_layer)\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(),\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "        reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=50, min_lr=0.0001)\n",
    "\n",
    "        file_path = self.output_directory + 'best_model.hdf5'\n",
    "\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss',\n",
    "                                                           save_best_only=True)\n",
    "\n",
    "        self.callbacks = [reduce_lr, model_checkpoint]\n",
    "\n",
    "        return model\n",
    "\n",
    "    def fit(self, x_train, y_train, x_val, y_val, y_true):\n",
    "        if not tf.test.is_gpu_available:\n",
    "            print('error')\n",
    "            exit()\n",
    "        # x_val and y_val are only used to monitor the test loss and NOT for training\n",
    "        batch_size = 64\n",
    "        nb_epochs = 500\n",
    "\n",
    "        mini_batch_size = int(min(x_train.shape[0] / 10, batch_size))\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        hist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,\n",
    "                              verbose=self.verbose, validation_data=(x_val, y_val), callbacks=self.callbacks)\n",
    "\n",
    "        duration = time.time() - start_time\n",
    "\n",
    "        self.model.save(self.output_directory + 'last_model.hdf5')\n",
    "\n",
    "        y_pred = self.predict(x_val, y_true, x_train, y_train, y_val,\n",
    "                              return_df_metrics=False)\n",
    "\n",
    "        # save predictions\n",
    "        np.save(self.output_directory + 'y_pred.npy', y_pred)\n",
    "\n",
    "        # convert the predicted from binary to integer\n",
    "        y_pred = np.argmax(y_pred, axis=1)\n",
    "\n",
    "        df_metrics = save_logs(self.output_directory, hist, y_pred, y_true, duration)\n",
    "\n",
    "        keras.backend.clear_session()\n",
    "\n",
    "        return df_metrics\n",
    "\n",
    "    def predict(self, x_test, y_true, x_train, y_train, y_test, return_df_metrics=True):\n",
    "        start_time = time.time()\n",
    "        model_path = self.output_directory + 'best_model.hdf5'\n",
    "        model = keras.models.load_model(model_path)\n",
    "        y_pred = model.predict(x_test)\n",
    "        if return_df_metrics:\n",
    "            y_pred = np.argmax(y_pred, axis=1)\n",
    "            df_metrics = calculate_metrics(y_true, y_pred, 0.0)\n",
    "            return df_metrics\n",
    "        else:\n",
    "            test_duration = time.time() - start_time\n",
    "            save_test_duration(self.output_directory + 'test_duration.csv', test_duration)\n",
    "            return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"### Encoder \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "class Classifier_ENCODER:\n",
    "\n",
    "    def __init__(self, output_directory, input_shape, nb_classes, verbose=False,build=True):\n",
    "        self.output_directory = output_directory\n",
    "        if build == True:\n",
    "            self.model = self.build_model(input_shape, nb_classes)\n",
    "            if (verbose == True):\n",
    "                self.model.summary()\n",
    "            self.verbose = verbose\n",
    "            self.model.save_weights(self.output_directory + 'model_init.hdf5')\n",
    "\n",
    "    def build_model(self, input_shape, nb_classes):\n",
    "        input_layer = keras.layers.Input(input_shape)\n",
    "\n",
    "        # conv block -1\n",
    "        conv1 = keras.layers.Conv1D(filters=128,kernel_size=5,strides=1,padding='same')(input_layer)\n",
    "        conv1 = tfa.layers.InstanceNormalization()(conv1)\n",
    "        conv1 = keras.layers.PReLU(shared_axes=[1])(conv1)\n",
    "        conv1 = keras.layers.Dropout(rate=0.2)(conv1)\n",
    "        conv1 = keras.layers.MaxPooling1D(pool_size=2)(conv1)\n",
    "        # conv block -2\n",
    "        conv2 = keras.layers.Conv1D(filters=256,kernel_size=11,strides=1,padding='same')(conv1)\n",
    "        conv2 = tfa.layers.InstanceNormalization()(conv2)\n",
    "        conv2 = keras.layers.PReLU(shared_axes=[1])(conv2)\n",
    "        conv2 = keras.layers.Dropout(rate=0.2)(conv2)\n",
    "        conv2 = keras.layers.MaxPooling1D(pool_size=2)(conv2)\n",
    "        # conv block -3\n",
    "        conv3 = keras.layers.Conv1D(filters=512,kernel_size=21,strides=1,padding='same')(conv2)\n",
    "        conv3 = tfa.layers.InstanceNormalization()(conv3)\n",
    "        conv3 = keras.layers.PReLU(shared_axes=[1])(conv3)\n",
    "        conv3 = keras.layers.Dropout(rate=0.2)(conv3)\n",
    "        # split for attention\n",
    "        attention_data = keras.layers.Lambda(lambda x: x[:,:,:256])(conv3)\n",
    "        attention_softmax = keras.layers.Lambda(lambda x: x[:,:,256:])(conv3)\n",
    "        # attention mechanism\n",
    "        attention_softmax = keras.layers.Softmax()(attention_softmax)\n",
    "        multiply_layer = keras.layers.Multiply()([attention_softmax,attention_data])\n",
    "        # last layer\n",
    "        dense_layer = keras.layers.Dense(units=256,activation='sigmoid')(multiply_layer)\n",
    "        dense_layer = tfa.layers.InstanceNormalization()(dense_layer)\n",
    "        # output layer\n",
    "        flatten_layer = keras.layers.Flatten()(dense_layer)\n",
    "        output_layer = keras.layers.Dense(units=nb_classes,activation='softmax')(flatten_layer)\n",
    "\n",
    "        model = keras.models.Model(inputs=input_layer, outputs=output_layer)\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(0.00001),\n",
    "                      metrics=['accuracy'])\n",
    "\n",
    "        file_path = self.output_directory + 'best_model.hdf5'\n",
    "\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path,\n",
    "                                                           monitor='loss', save_best_only=True)\n",
    "\n",
    "        self.callbacks = [model_checkpoint]\n",
    "\n",
    "        return model\n",
    "\n",
    "    def fit(self, x_train, y_train, x_val, y_val, y_true):\n",
    "        if not tf.test.is_gpu_available:\n",
    "            print('error')\n",
    "            exit()\n",
    "        # x_val and y_val are only used to monitor the test loss and NOT for training\n",
    "        batch_size = 12\n",
    "        nb_epochs = 100\n",
    "\n",
    "        mini_batch_size = batch_size\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        hist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,\n",
    "                              verbose=self.verbose, validation_data=(x_val, y_val), callbacks=self.callbacks)\n",
    "\n",
    "        duration = time.time() - start_time\n",
    "\n",
    "        self.model.save(self.output_directory+'last_model.hdf5')\n",
    "\n",
    "        model = keras.models.load_model(self.output_directory + 'best_model.hdf5')\n",
    "\n",
    "        y_pred = model.predict(x_val)\n",
    "\n",
    "        # convert the predicted from binary to integer\n",
    "        y_pred = np.argmax(y_pred, axis=1)\n",
    "\n",
    "        save_logs(self.output_directory, hist, y_pred, y_true, duration, lr=False)\n",
    "\n",
    "        keras.backend.clear_session()\n",
    "\n",
    "    def predict(self, x_test,y_true,x_train,y_train,y_test,return_df_metrics = True):\n",
    "        model_path = self.output_directory + 'best_model.hdf5'\n",
    "        model = keras.models.load_model(model_path)\n",
    "        y_pred = model.predict(x_test)\n",
    "        if return_df_metrics:\n",
    "            y_pred = np.argmax(y_pred, axis=1)\n",
    "            df_metrics = calculate_metrics(y_true, y_pred, 0.0)\n",
    "            return df_metrics\n",
    "        else:\n",
    "            return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"## MCNN Model\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier_MCNN:\n",
    "\n",
    "    def __init__(self, output_directory, verbose, build=True):\n",
    "        self.output_directory = output_directory\n",
    "        self.verbose = verbose\n",
    "        #self.pool_factors = [2,3,5] # used for hyperparameters grid search\n",
    "        self.pool_factors = [2, 3, 5] # used for hyperparameters grid search\n",
    "        self.filter_sizes = [0.05,0.1,0.2] # used for hyperparameters grid search \n",
    "\n",
    "    def slice_data(self, data_x, data_y, slice_ratio): \n",
    "        n = data_x.shape[0]\n",
    "        length = data_x.shape[1]\n",
    "        n_dim = data_x.shape[2] # for MTS \n",
    "        nb_classes = data_y.shape[1]\n",
    "\n",
    "        length_sliced = int(length * slice_ratio)\n",
    "       \n",
    "        increase_num = length - length_sliced + 1 #if increase_num =5, it means one ori becomes 5 new instances.\n",
    "        n_sliced = n * increase_num\n",
    "\n",
    "        new_x = np.zeros((n_sliced, length_sliced,n_dim))\n",
    "        new_y = np.zeros((n_sliced,nb_classes))\n",
    "        for i in range(n):\n",
    "            for j in range(increase_num):\n",
    "                new_x[i * increase_num + j, :,:] = data_x[i,j : j + length_sliced,:]\n",
    "                new_y[i * increase_num + j] = np.int_(data_y[i].astype(np.float32))\n",
    "\n",
    "        return new_x, new_y\n",
    "\n",
    "    def split_train(self, train_x,train_y):\n",
    "        #shuffle for splitting train set and dataset\n",
    "        n = train_x.shape[0]\n",
    "        ind = np.arange(n)\n",
    "        np.random.shuffle(ind) #shuffle the train set\n",
    "        \n",
    "        #split train set into train set and validation set\n",
    "        valid_x = train_x[ind[0:int(0.2 * n)]]\n",
    "        valid_y = train_y[ind[0:int(0.2 * n)]]\n",
    "\n",
    "        ind = np.delete(ind, (range(0,int(0.2 * n))))\n",
    "\n",
    "        train_x = train_x[ind] \n",
    "        train_y = train_y[ind]\n",
    "            \n",
    "        return train_x,train_y,valid_x,valid_y\n",
    "\n",
    "    def _downsample(self, data_x, sample_rate, offset = 0):\n",
    "        num = data_x.shape[0]\n",
    "        length_x = data_x.shape[1]\n",
    "        num_dim = data_x.shape[2] # for MTS \n",
    "        last_one = 0\n",
    "        if length_x % sample_rate > offset:\n",
    "            last_one = 1\n",
    "        new_length = int(np.floor( length_x / sample_rate)) + last_one\n",
    "        output = np.zeros((num, new_length,num_dim))\n",
    "        for i in range(new_length):\n",
    "            output[:,i] = np.array(data_x[:,offset + sample_rate * i])\n",
    "\n",
    "        return output\n",
    "\n",
    "    def _movingavrg(self, data_x, window_size):\n",
    "        num = data_x.shape[0]\n",
    "        length_x = data_x.shape[1]\n",
    "        num_dim = data_x.shape[2] # for MTS \n",
    "        output_len = length_x - window_size + 1\n",
    "        output = np.zeros((num, output_len,num_dim))\n",
    "        for i in range(output_len):\n",
    "            output[:,i] = np.mean(data_x[:, i : i + window_size], axis = 1)\n",
    "        return output\n",
    "\n",
    "    def movingavrg(self, data_x, window_base, step_size, num):\n",
    "        if num == 0:\n",
    "            return (None, [])\n",
    "        out = self._movingavrg(data_x, window_base)\n",
    "        data_lengths = [out.shape[1]]\n",
    "        for i in range(1, num):\n",
    "            window_size = window_base + step_size * i\n",
    "            if window_size > data_x.shape[1]:\n",
    "                continue\n",
    "            new_series = self._movingavrg(data_x, window_size)\n",
    "            data_lengths.append( new_series.shape[1] )\n",
    "            out = np.concatenate([out, new_series], axis = 1)\n",
    "        return (out, data_lengths)\n",
    "\n",
    "    def batch_movingavrg(self, train,valid,test, window_base, step_size, num):\n",
    "        (new_train, lengths) = self.movingavrg(train, window_base, step_size, num)\n",
    "        (new_valid, lengths) = self.movingavrg(valid, window_base, step_size, num)\n",
    "        (new_test, lengths) = self.movingavrg(test, window_base, step_size, num)\n",
    "        return (new_train, new_valid, new_test, lengths)\n",
    "\n",
    "    def downsample(self, data_x, base, step_size, num):\n",
    "        # the case for dataset JapaneseVowels MTS\n",
    "        if data_x.shape[1] ==26 :\n",
    "            return (None,[]) # too short to apply downsampling\n",
    "        if num == 0:\n",
    "            return (None, [])\n",
    "        out = self._downsample(data_x, base,0)\n",
    "        data_lengths = [out.shape[1]]\n",
    "        #for offset in range(1,base): #for the base case\n",
    "        #    new_series = _downsample(data_x, base, offset)\n",
    "        #    data_lengths.append( new_series.shape[1] )\n",
    "        #    out = np.concatenate( [out, new_series], axis = 1)\n",
    "        for i in range(1, num):\n",
    "            sample_rate = base + step_size * i \n",
    "            if sample_rate > data_x.shape[1]:\n",
    "                continue\n",
    "            for offset in range(0,1):#sample_rate):\n",
    "                new_series = self._downsample(data_x, sample_rate, offset)\n",
    "                data_lengths.append( new_series.shape[1] )\n",
    "                out = np.concatenate( [out, new_series], axis = 1)\n",
    "        return (out, data_lengths)\n",
    "\n",
    "    def batch_downsample(self, train,valid,test, window_base, step_size, num):\n",
    "        (new_train, lengths) = self.downsample(train, window_base, step_size, num)\n",
    "        (new_valid, lengths) = self.downsample(valid, window_base, step_size, num)\n",
    "        (new_test, lengths) = self.downsample(test, window_base, step_size, num)\n",
    "        return (new_train, new_valid, new_test, lengths)\n",
    "\n",
    "    def get_pool_factor(self,conv_shape,pool_size):\n",
    "        for pool_factor in self.pool_factors:\n",
    "            temp_pool_size = int(int(conv_shape)/pool_factor)\n",
    "            print(temp_pool_size)\n",
    "            if temp_pool_size == pool_size:\n",
    "                return pool_factor\n",
    "\n",
    "        raise Exception('Error on pool factor')\n",
    "\n",
    "    def train(self, x_train, y_train, x_test, y_test,y_true, pool_factor=None, filter_size=None,do_train=True):\n",
    "        window_size = 0.2\n",
    "        n_train_batch = 10\n",
    "        n_epochs = 200\n",
    "        max_train_batch_size = 256\n",
    "\n",
    "        # print('Original train shape: ', x_train.shape)\n",
    "        # print('Original test shape: ', x_test.shape)\n",
    "\n",
    "        # split train into validation set with validation_size = 0.2 train_size \n",
    "        x_train,y_train,x_val,y_val = self.split_train(x_train,y_train)\n",
    "        \n",
    "        ori_len = x_train.shape[1] # original_length of time series  \n",
    "        slice_ratio = 0.9\n",
    "\n",
    "        if do_train == True:\n",
    "            kernel_size = int(ori_len * filter_size)\n",
    "\n",
    "        if do_train == False:\n",
    "            model = keras.models.load_model(self.output_directory+'best_model.hdf5')\n",
    "\n",
    "            # model.summary()\n",
    "\n",
    "            pool_size = model.get_layer('max_pooling1d_1').get_config()['pool_size'][0]\n",
    "\n",
    "            conv_shape = model.get_layer('conv1d_1').output_shape[1]\n",
    "\n",
    "\n",
    "            pool_factor = self.get_pool_factor(conv_shape,pool_size)\n",
    "\n",
    "        #restrict slice ratio when data lenght is too large\n",
    "        if ori_len > 500 : \n",
    "            slice_ratio = slice_ratio if slice_ratio > 0.98 else 0.98\n",
    "        elif ori_len < 16:\n",
    "            slice_ratio = 0.7\n",
    "\n",
    "        increase_num = ori_len - int(ori_len * slice_ratio) + 1 #this can be used as the bath size\n",
    "\n",
    "        train_batch_size = int(x_train.shape[0] * increase_num / n_train_batch)\n",
    "        if train_batch_size > max_train_batch_size : \n",
    "            # limit the train_batch_size \n",
    "            n_train_batch = int(x_train.shape[0] * increase_num / max_train_batch_size)\n",
    "        \n",
    "        # data augmentation by slicing the length of the series \n",
    "        x_train,y_train = self.slice_data(x_train,y_train,slice_ratio)\n",
    "        x_val,y_val = self.slice_data(x_val,y_val,slice_ratio)\n",
    "        x_test,y_test = self.slice_data(x_test,y_test,slice_ratio)\n",
    "\n",
    "        train_set_x, train_set_y = x_train,y_train\n",
    "        valid_set_x, valid_set_y = x_val,y_val\n",
    "        test_set_x, _ = x_test,y_test\n",
    "\n",
    "        valid_num = valid_set_x.shape[0]\n",
    "        \n",
    "        # print(\"increase factor is \", increase_num, ', ori len', ori_len)\n",
    "        valid_num_batch = int(valid_num / increase_num)\n",
    "\n",
    "        test_num = test_set_x.shape[0]\n",
    "        test_num_batch = int(test_num / increase_num)\n",
    "\n",
    "        length_train = train_set_x.shape[1] #length after slicing.\n",
    "        \n",
    "        window_size = int(length_train * window_size) if window_size < 1 else int(window_size)\n",
    "\n",
    "        #*******set up the ma and ds********#\n",
    "        ma_base,ma_step,ma_num   = 5, 6, 1\n",
    "        ds_base,ds_step, ds_num  = 2, 1, 4\n",
    "\n",
    "        ds_num_max = length_train / (pool_factor * window_size)\n",
    "        ds_num = int(min(ds_num, ds_num_max))\n",
    "        \n",
    "        #*******set up the ma and ds********#\n",
    "\n",
    "        (ma_train, ma_valid, ma_test , ma_lengths) = self.batch_movingavrg(train_set_x,\n",
    "                                                        valid_set_x, test_set_x,\n",
    "                                                        ma_base, ma_step, ma_num)\n",
    "        (ds_train, ds_valid, ds_test , ds_lengths) = self.batch_downsample(train_set_x,\n",
    "                                                        valid_set_x, test_set_x,\n",
    "                                                        ds_base, ds_step, ds_num)\n",
    "\n",
    "        #concatenate directly\n",
    "        data_lengths = [length_train] \n",
    "        #downsample part:\n",
    "        if ds_lengths != []:\n",
    "            data_lengths +=  ds_lengths\n",
    "            train_set_x = np.concatenate([train_set_x, ds_train], axis = 1)\n",
    "            valid_set_x = np.concatenate([valid_set_x, ds_valid], axis = 1)\n",
    "            test_set_x = np.concatenate([test_set_x, ds_test], axis = 1)\n",
    "\n",
    "        #moving average part\n",
    "        if ma_lengths != []:\n",
    "            data_lengths += ma_lengths\n",
    "            train_set_x = np.concatenate([train_set_x, ma_train], axis = 1)\n",
    "            valid_set_x = np.concatenate([valid_set_x, ma_valid], axis = 1)\n",
    "            test_set_x = np.concatenate([test_set_x, ma_test], axis = 1)\n",
    "        # print(\"Data length:\", data_lengths)\n",
    "\n",
    "        n_train_size = train_set_x.shape[0]\n",
    "        n_valid_size = valid_set_x.shape[0]\n",
    "        n_test_size = test_set_x.shape[0]\n",
    "        batch_size = int(n_train_size / n_train_batch)\n",
    "        n_train_batches = int(n_train_size / batch_size)\n",
    "        data_dim = train_set_x.shape[1]  \n",
    "        num_dim = train_set_x.shape[2] # For MTS \n",
    "        nb_classes = train_set_y.shape[1] \n",
    "\n",
    "        # print('train size', n_train_size, ',valid size', n_valid_size, ' test size', n_test_size)\n",
    "        # print('batch size ', batch_size)\n",
    "        # print('n_train_batches is ', n_train_batches)\n",
    "        # print('data dim is ', data_dim)\n",
    "        # print('---------------------------')\n",
    "\n",
    "        ######################\n",
    "        # BUILD ACTUAL MODEL #\n",
    "        ######################\n",
    "        # print('building the model...')\n",
    "\n",
    "        input_shapes, max_length = self.get_list_of_input_shapes(data_lengths,num_dim)\n",
    "\n",
    "        start_time = time.time()\n",
    "\n",
    "        best_validation_loss = np.inf\n",
    "\n",
    "        if do_train == True:\n",
    "\n",
    "            model = self.build_model(input_shapes, nb_classes, pool_factor, kernel_size)\n",
    "\n",
    "            if (self.verbose==True) :\n",
    "                model.summary()\n",
    "\n",
    "            # print('Training')\n",
    "\n",
    "\n",
    "            # early-stopping parameters\n",
    "            patience = 10000  # look as this many examples regardless\n",
    "            patience_increase = 2  # wait this much longer when a new best is\n",
    "                                   # found\n",
    "            improvement_threshold = 0.995  # a relative improvement of this much is\n",
    "                                           # considered significant\n",
    "            validation_frequency = min(n_train_batches, patience / 2)\n",
    "                                          # go through this many\n",
    "                                          # minibatche before checking the network\n",
    "                                          # on the validation set; in this case we\n",
    "                                          # check every epoch\n",
    "            max_before_stopping = 500\n",
    "\n",
    "            best_iter = 0\n",
    "            valid_loss = 0.\n",
    "\n",
    "            epoch = 0\n",
    "            done_looping = False\n",
    "            num_no_update_epoch = 0\n",
    "            epoch_avg_cost = float('inf')\n",
    "            epoch_avg_err = float('inf')\n",
    "\n",
    "            while (epoch < n_epochs) and (not done_looping):\n",
    "                epoch = epoch + 1\n",
    "                epoch_train_err = 0.\n",
    "                epoch_cost = 0.\n",
    "\n",
    "                num_no_update_epoch += 1\n",
    "                if num_no_update_epoch == max_before_stopping:\n",
    "                    break\n",
    "\n",
    "\n",
    "                for minibatch_index in range(n_train_batches):\n",
    "\n",
    "                    iteration = (epoch - 1) * n_train_batches + minibatch_index\n",
    "\n",
    "                    x = train_set_x[minibatch_index*batch_size: (minibatch_index+1)*batch_size]\n",
    "                    y = train_set_y[minibatch_index*batch_size: (minibatch_index+1)*batch_size]\n",
    "\n",
    "                    x = self.split_input_for_model(x,input_shapes)\n",
    "\n",
    "                    cost_ij, accuracy = model.train_on_batch(x,y)\n",
    "\n",
    "                    train_err = 1 - accuracy\n",
    "\n",
    "                    epoch_train_err = epoch_train_err + train_err\n",
    "                    epoch_cost = epoch_cost + cost_ij\n",
    "\n",
    "\n",
    "                    if (iteration + 1) % validation_frequency == 0:\n",
    "\n",
    "                        valid_losses = []\n",
    "                        for i in range(valid_num_batch):\n",
    "                            x = valid_set_x[i * (increase_num) : (i + 1) * (increase_num)]\n",
    "                            y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))\n",
    "\n",
    "                            # convert the predicted from binary to integer\n",
    "                            y_pred = np.argmax(y_pred , axis=1)\n",
    "                            label = np.argmax(valid_set_y[i * increase_num])\n",
    "\n",
    "                            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)\n",
    "                            unique_value = unique_value.tolist()\n",
    "\n",
    "                            curr_err = 1.\n",
    "                            if label in unique_value:\n",
    "                                target_ind = unique_value.index(label)\n",
    "                                count = count.tolist()\n",
    "                                sorted_count = sorted(count)\n",
    "                                if count[target_ind] == sorted_count[-1]:\n",
    "                                    if len(sorted_count) > 1 and sorted_count[-1] == sorted_count[-2]:\n",
    "                                        curr_err = 0.5 #tie\n",
    "                                    else:\n",
    "                                        curr_err = 0\n",
    "                            valid_losses.append(curr_err)\n",
    "                        valid_loss = sum(valid_losses) / float(len(valid_losses))\n",
    "\n",
    "                        # print('...epoch%i,valid err: %.5f |' % (epoch,valid_loss))\n",
    "\n",
    "                        # if we got the best validation score until now\n",
    "                        if valid_loss <= best_validation_loss:\n",
    "                            num_no_update_epoch = 0\n",
    "\n",
    "                            #improve patience if loss improvement is good enough\n",
    "                            if valid_loss < best_validation_loss*improvement_threshold:\n",
    "                                patience = max(patience,iteration*patience_increase)\n",
    "\n",
    "                            # save best validation score and iteration number\n",
    "                            best_validation_loss = valid_loss\n",
    "                            best_iter = iteration\n",
    "\n",
    "                            # save model in h5 format\n",
    "                            model.save(self.output_directory+'best_model.hdf5')\n",
    "\n",
    "                        model.save(self.output_directory + 'last_model.hdf5')\n",
    "                    if patience<= iteration:\n",
    "                        done_looping=True\n",
    "                        break\n",
    "                epoch_avg_cost = epoch_cost/n_train_batches\n",
    "                epoch_avg_err = epoch_train_err/n_train_batches\n",
    "\n",
    "                # print ('train err %.5f, cost %.4f' %(epoch_avg_err,epoch_avg_cost))\n",
    "                if epoch_avg_cost == 0:\n",
    "                    break\n",
    "\n",
    "            # print('Optimization complete.')\n",
    "\n",
    "        # test the model\n",
    "        # print('Testing')\n",
    "        # load best model\n",
    "        model = keras.models.load_model(self.output_directory+'best_model.hdf5')\n",
    "\n",
    "        # get the true predictions of the test set\n",
    "        y_predicted = []\n",
    "        for i in range(test_num_batch):\n",
    "            x = test_set_x[i * (increase_num) : (i + 1) * (increase_num)]\n",
    "            y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))\n",
    "\n",
    "            # convert the predicted from binary to integer \n",
    "            y_pred = np.argmax(y_pred , axis=1)\n",
    "\n",
    "            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)\n",
    "\n",
    "            idx_max = np.argmax(count)\n",
    "            predicted_label = unique_value[idx_max]\n",
    "\n",
    "            y_predicted.append(predicted_label)\n",
    "\n",
    "        y_pred = np.array(y_predicted)\n",
    "\n",
    "        duration = time.time() - start_time        \n",
    "\n",
    "        df_metrics = calculate_metrics(y_true,y_pred, duration)\n",
    "\n",
    "        # print(y_true.shape)\n",
    "        # print(y_pred.shape)\n",
    "\n",
    "        df_metrics.to_csv(self.output_directory+'df_metrics.csv', index=False)\n",
    "\n",
    "        return df_metrics, model , best_validation_loss\n",
    "\n",
    "\n",
    "    def split_input_for_model(self, x, input_shapes):\n",
    "        res = []\n",
    "        indx = 0 \n",
    "        for input_shape in input_shapes:\n",
    "            res.append(x[:,indx:indx+input_shape[0],:])\n",
    "            indx = indx + input_shape[0]\n",
    "        return res\n",
    "\n",
    "    def build_model(self, input_shapes, nb_classes, pool_factor, kernel_size):\n",
    "        input_layers = []\n",
    "        stage_1_layers = []\n",
    "\n",
    "        for input_shape in input_shapes: \n",
    "\n",
    "            input_layer = keras.layers.Input(input_shape)\n",
    "            \n",
    "            input_layers.append(input_layer)\n",
    "\n",
    "            conv_layer = keras.layers.Conv1D(filters=256, kernel_size=kernel_size, padding='same', \n",
    "                activation='sigmoid', kernel_initializer='glorot_uniform')(input_layer)\n",
    "\n",
    "            # should all concatenated have the same length\n",
    "            pool_size = int(int(conv_layer.shape[1])/pool_factor)\n",
    "\n",
    "            max_layer = keras.layers.MaxPooling1D(pool_size=pool_size)(conv_layer)\n",
    "\n",
    "            # max_layer = keras.layers.GlobalMaxPooling1D()(conv_layer)\n",
    "\n",
    "            stage_1_layers.append(max_layer)\n",
    "\n",
    "        concat_layer = keras.layers.Concatenate(axis=-1)(stage_1_layers)\n",
    "\n",
    "        kernel_size = int(min(kernel_size, int(concat_layer.shape[1]))) # kernel shouldn't exceed the length  \n",
    "\n",
    "        full_conv = keras.layers.Conv1D(filters=256, kernel_size=kernel_size, padding='same', \n",
    "            activation='sigmoid', kernel_initializer='glorot_uniform')(concat_layer)\n",
    "\n",
    "        pool_size = int(int(full_conv.shape[1])/pool_factor)\n",
    "\n",
    "        full_max = keras.layers.MaxPooling1D(pool_size=pool_size)(full_conv)\n",
    "\n",
    "        full_max = keras.layers.Flatten()(full_max) \n",
    "\n",
    "        fully_connected = keras.layers.Dense(units=256, activation='sigmoid', \n",
    "            kernel_initializer='glorot_uniform')(full_max)\n",
    "\n",
    "        output_layer = keras.layers.Dense(units=nb_classes, activation='softmax', \n",
    "            kernel_initializer='glorot_uniform')(fully_connected)\n",
    "\n",
    "        model = keras.models.Model(inputs=input_layers, outputs=output_layer)\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(lr=0.1),\n",
    "            metrics=['accuracy'])\n",
    "        \n",
    "        return model \n",
    "\n",
    "    def get_list_of_input_shapes(self, data_lengths, num_dim):\n",
    "        input_shapes = []\n",
    "        max_length = 0\n",
    "        for i in data_lengths: \n",
    "            input_shapes.append((i,num_dim))\n",
    "            max_length = max(max_length, i)\n",
    "        return input_shapes , max_length\n",
    "\n",
    "    def fit(self, x_train, y_train, x_test, y_test,y_true):\n",
    "        if not tf.test.is_gpu_available:\n",
    "            print('error')\n",
    "            exit()\n",
    "        best_df_metrics = None\n",
    "        best_valid_loss = np.inf\n",
    "\n",
    "        output_directory_root = self.output_directory \n",
    "        # grid search \n",
    "        for pool_factor in self.pool_factors:\n",
    "            for filter_size in self.filter_sizes:\n",
    "                self.output_directory = output_directory_root+'/hyper_param_search/'+'/pool_factor_'+ \\\n",
    "                    str(pool_factor)+'/filter_size_'+str(filter_size)+'/' \n",
    "                create_directory(self.output_directory)\n",
    "                df_metrics, model , valid_loss = self.train(x_train, y_train, x_test, \n",
    "                                                          y_test,y_true,pool_factor,filter_size)\n",
    "\n",
    "                if (valid_loss < best_valid_loss): \n",
    "                    best_valid_loss = valid_loss\n",
    "                    best_df_metrics = df_metrics\n",
    "                    best_df_metrics.to_csv(output_directory_root+'df_metrics.csv', index=False)\n",
    "                    model.save(output_directory_root+'best_model.hdf5')\n",
    "\n",
    "                model = None\n",
    "                # clear memeory \n",
    "                keras.backend.clear_session()\n",
    "\n",
    "    def predict(self, x_test, y_true,x_train,y_train,y_test):\n",
    "        df_metrics, _ , _ = self.train(x_train, y_train, x_test, y_test,y_true, do_train=False)\n",
    "\n",
    "        return df_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"## t-LeNet Model\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier_TLENET:\n",
    "    \n",
    "    def __init__(self, output_directory, verbose,build=True):\n",
    "        self.output_directory = output_directory\n",
    "        self.verbose = verbose\n",
    "        self.warping_ratios = [0.5,1,2]\n",
    "        self.slice_ratio = 0.1\n",
    "        \n",
    "    def slice_data(self, data_x, data_y, length_sliced): \n",
    "        n = data_x.shape[0]\n",
    "        length = data_x.shape[1]\n",
    "        n_dim = data_x.shape[2] # for MTS \n",
    "        nb_classes = data_y.shape[1]\n",
    "       \n",
    "        increase_num = length - length_sliced + 1 #if increase_num =5, it means one ori becomes 5 new instances.\n",
    "        n_sliced = n * increase_num\n",
    "\n",
    "        print((n_sliced, length_sliced,n_dim))\n",
    "\n",
    "        new_x = np.zeros((n_sliced, length_sliced,n_dim))\n",
    "        new_y = np.zeros((n_sliced,nb_classes))\n",
    "        for i in range(n):\n",
    "            for j in range(increase_num):\n",
    "                new_x[i * increase_num + j, :,:] = data_x[i,j : j + length_sliced,:]\n",
    "                new_y[i * increase_num + j] = np.int_(data_y[i].astype(np.float32))\n",
    "\n",
    "        return new_x, new_y, increase_num \n",
    "\n",
    "    def window_warping(self, data_x, warping_ratio): \n",
    "        num_x = data_x.shape[0]\n",
    "        len_x = data_x.shape[1]\n",
    "        dim_x = data_x.shape[2]\n",
    "        \n",
    "        x = np.arange(0,len_x,warping_ratio)\n",
    "        xp = np.arange(0,len_x)\n",
    "        \n",
    "        new_length = len(np.interp(x,xp,data_x[0,:,0]))\n",
    "        \n",
    "        warped_series = np.zeros((num_x,new_length,dim_x),dtype=np.float64)\n",
    "        \n",
    "        for i in range(num_x):\n",
    "            for j in range(dim_x):\n",
    "                warped_series[i,:,j] = np.interp(x,xp,data_x[i,:,j])\n",
    "                \n",
    "        return warped_series\n",
    "    \n",
    "    def build_model(self, input_shape, nb_classes):\n",
    "        input_layer = keras.layers.Input(input_shape)\n",
    "        \n",
    "        conv_1 = keras.layers.Conv1D(filters=5,kernel_size=5,activation='relu', padding='same')(input_layer)\n",
    "        conv_1 = keras.layers.MaxPool1D(pool_size=2)(conv_1)\n",
    "        \n",
    "        conv_2 = keras.layers.Conv1D(filters=20, kernel_size=5, activation='relu', padding='same')(conv_1)\n",
    "        conv_2 = keras.layers.MaxPool1D(pool_size=4)(conv_2)\n",
    "        \n",
    "        # they did not mention the number of hidden units in the fully-connected layer\n",
    "        # so we took the lenet they referenced \n",
    "        \n",
    "        flatten_layer = keras.layers.Flatten()(conv_2)\n",
    "        fully_connected_layer = keras.layers.Dense(500,activation='relu')(flatten_layer)\n",
    "        \n",
    "        output_layer = keras.layers.Dense(nb_classes,activation='softmax')(fully_connected_layer)\n",
    "        \n",
    "        model = keras.models.Model(inputs=input_layer,outputs=output_layer)\n",
    "        \n",
    "        model.compile(optimizer=keras.optimizers.Adam(lr=0.01,decay=0.005),\n",
    "                      loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "        \n",
    "        file_path = self.output_directory+'best_model.hdf5'\n",
    "\n",
    "        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', \n",
    "            save_best_only=True)\n",
    "        \n",
    "        self.callbacks=[model_checkpoint]\n",
    "        \n",
    "        return model\n",
    "    \n",
    "    def pre_processing(self,x_train,y_train,x_test,y_test):\n",
    "        length_ratio = int(self.slice_ratio*x_train.shape[1])\n",
    "        \n",
    "        x_train_augmented = [] # list of the augmented as well as the original data\n",
    "        x_test_augmented = [] # list of the augmented as well as the original data\n",
    "        \n",
    "        y_train_augmented = []\n",
    "        y_test_augmented = [] \n",
    "        \n",
    "        # data augmentation using WW\n",
    "        for warping_ratio in self.warping_ratios:\n",
    "            x_train_augmented.append(self.window_warping(x_train,warping_ratio))\n",
    "            x_test_augmented.append(self.window_warping(x_test,warping_ratio))\n",
    "            y_train_augmented.append(y_train)\n",
    "            y_test_augmented.append(y_test)\n",
    "                \n",
    "        increase_nums = []\n",
    "        \n",
    "        # data augmentation using WS \n",
    "        for i in range(0,len(x_train_augmented)):\n",
    "            x_train_augmented[i],y_train_augmented[i],increase_num = self.slice_data(\n",
    "                    x_train_augmented[i],y_train,length_ratio)\n",
    "            x_test_augmented[i],y_test_augmented[i],increase_num = self.slice_data(\n",
    "                    x_test_augmented[i],y_test,length_ratio)\n",
    "            increase_nums.append(increase_num)\n",
    "        \n",
    "        tot_increase_num = np.array(increase_nums).sum()\n",
    "        \n",
    "        new_x_train = np.zeros((x_train.shape[0]*tot_increase_num, length_ratio,x_train.shape[2]))\n",
    "        new_y_train = np.zeros((y_train.shape[0]*tot_increase_num,y_train.shape[1]))\n",
    "        \n",
    "        new_x_test = np.zeros((x_test.shape[0]*tot_increase_num, length_ratio,x_test.shape[2]))\n",
    "        new_y_test = np.zeros((y_test.shape[0]*tot_increase_num,y_test.shape[1]))\n",
    "        \n",
    "        # merge the list of augmented data\n",
    "        idx = 0\n",
    "        for i in range(x_train.shape[0]):\n",
    "            for j in range(len(increase_nums)):\n",
    "                increase_num = increase_nums[j]\n",
    "                new_x_train [idx:idx+increase_num ,:,:] = \\\n",
    "                    x_train_augmented[j][i*increase_num:(i+1)*increase_num,:,:] \n",
    "                new_y_train [idx:idx+increase_num,:] = \\\n",
    "                    y_train_augmented[j][i*increase_num:(i+1)*increase_num,:]\n",
    "                idx += increase_num\n",
    "                \n",
    "        # do the same for the test set \n",
    "        idx = 0\n",
    "        for i in range(x_test.shape[0]):\n",
    "            for j in range(len(increase_nums)):\n",
    "                increase_num = increase_nums[j]\n",
    "                new_x_test [idx:idx+increase_num ,:,:] = \\\n",
    "                    x_test_augmented[j][i*increase_num:(i+1)*increase_num,:,:] \n",
    "                new_y_test [idx:idx+increase_num,:] = \\\n",
    "                    y_test_augmented[j][i*increase_num:(i+1)*increase_num,:]\n",
    "                idx += increase_num\n",
    "        return new_x_train,new_y_train,new_x_test,new_y_test, tot_increase_num\n",
    "    \n",
    "    def fit(self, x_train, y_train, x_test, y_test,y_true):\n",
    "        if not tf.test.is_gpu_available:\n",
    "            print('error')\n",
    "            exit()\n",
    "        nb_epochs = 500\n",
    "        batch_size= 256\n",
    "        nb_classes = y_train.shape[1]\n",
    "\n",
    "        # limit the number of augmented time series if series too long or too many \n",
    "        if x_train.shape[1] > 500 or x_train.shape[0] > 2000 or x_test.shape[0] > 2000 : \n",
    "            self.warping_ratios = [1]\n",
    "            self.slice_ratio = 0.9\n",
    "        # increase the slice if series too short \n",
    "        if x_train.shape[1]*self.slice_ratio < 8:\n",
    "            self.slice_ratio = 8/x_train.shape[1]\n",
    "\n",
    "        ####################\n",
    "        ## pre-processing ##\n",
    "        ####################\n",
    "        \n",
    "        \n",
    "        x_train , y_train , x_test , y_test, tot_increase_num = self.pre_processing(x_train,y_train,x_test,y_test)\n",
    "        \n",
    "        print('Total increased number for each MTS: ',tot_increase_num)\n",
    "\n",
    "        #########################\n",
    "        ## done pre-processing ##\n",
    "        #########################\n",
    "        \n",
    "        input_shape = x_train.shape[1:]\n",
    "        model = self.build_model(input_shape,nb_classes)\n",
    "        \n",
    "        if self.verbose == True:\n",
    "            model.summary()\n",
    "        \n",
    "        start_time = time.time()\n",
    "        \n",
    "        hist = model.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs,\n",
    "            verbose=self.verbose, validation_data = (x_test,y_test),callbacks=self.callbacks)\n",
    "\n",
    "        model.save(self.output_directory+'last_model.hdf5')\n",
    "\n",
    "        model = keras.models.load_model(self.output_directory+'best_model.hdf5')\n",
    "        \n",
    "        y_pred = model.predict(x_test,batch_size=batch_size)\n",
    "        # convert the predicted from binary to integer \n",
    "        y_pred = np.argmax(y_pred , axis=1)\n",
    "        \n",
    "        # get the true predictions of the test set\n",
    "        y_predicted = []\n",
    "        test_num_batch = int(x_test.shape[0]/tot_increase_num)\n",
    "        for i in range(test_num_batch):\n",
    "            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred[i*tot_increase_num:(i+1)*tot_increase_num], True, True, True)\n",
    "\n",
    "            idx_max = np.argmax(count)\n",
    "            predicted_label = unique_value[idx_max]\n",
    "\n",
    "            y_predicted.append(predicted_label)\n",
    "\n",
    "        y_pred = np.array(y_predicted)\n",
    "        \n",
    "        duration = time.time() - start_time\n",
    "        \n",
    "        save_logs(self.output_directory, hist, y_pred, y_true, duration )\n",
    "\n",
    "        keras.backend.clear_session()\n",
    "\n",
    "    def predict(self, x_test, y_true,x_train,y_train,y_test):\n",
    "        batch_size = 256\n",
    "\n",
    "        # limit the number of augmented time series if series too long or too many\n",
    "        if x_train.shape[1] > 500 or x_train.shape[0] > 2000 or x_test.shape[0] > 2000:\n",
    "            self.warping_ratios = [1]\n",
    "            self.slice_ratio = 0.9\n",
    "        # increase the slice if series too short\n",
    "        if x_train.shape[1] * self.slice_ratio < 8:\n",
    "            self.slice_ratio = 8 / x_train.shape[1]\n",
    "\n",
    "        new_x_train, new_y_train, new_x_test, new_y_test, tot_increase_num = \\\n",
    "            self.pre_processing(x_train, y_train, x_test, y_test)\n",
    "\n",
    "        model_path = self.output_directory + 'best_model.hdf5'\n",
    "        model = keras.models.load_model(model_path)\n",
    "\n",
    "        y_pred = model.predict(new_x_test, batch_size=batch_size)\n",
    "        # convert the predicted from binary to integer\n",
    "        y_pred = np.argmax(y_pred, axis=1)\n",
    "\n",
    "        # get the true predictions of the test set\n",
    "        y_predicted = []\n",
    "        test_num_batch = int(new_x_test.shape[0] / tot_increase_num)\n",
    "        for i in range(test_num_batch):\n",
    "            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)\n",
    "\n",
    "            idx_max = np.argmax(count)\n",
    "            predicted_label = unique_value[idx_max]\n",
    "\n",
    "            y_predicted.append(predicted_label)\n",
    "\n",
    "        y_pred = np.array(y_predicted)\n",
    "\n",
    "        df_metrics = calculate_metrics(y_true, y_pred, 0.0)\n",
    "        return df_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"## Utility Functions\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_directory(directory_path):\n",
    "    if os.path.exists(directory_path):\n",
    "        return None\n",
    "    else:\n",
    "        try:\n",
    "            os.makedirs(directory_path)\n",
    "        except:\n",
    "            # in case another machine created the path meanwhile !:(\n",
    "            return None\n",
    "        return directory_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "def create_path(root_dir, classifier_name, archive_name):\n",
    "    output_directory = root_dir + '/results/' + classifier_name + '/' + archive_name + '/'\n",
    "    if os.path.exists(output_directory):\n",
    "        return None\n",
    "    else:\n",
    "        os.makedirs(output_directory)\n",
    "        return output_directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_metrics(y_true, y_pred, duration, y_true_val=None, y_pred_val=None):\n",
    "    res = pd.DataFrame(data=np.zeros((1, 4), dtype=np.float), index=[0],\n",
    "                       columns=['precision', 'accuracy', 'recall', 'duration'])\n",
    "    res['precision'] = precision_score(y_true, y_pred, average='macro')\n",
    "    res['accuracy'] = accuracy_score(y_true, y_pred)\n",
    "\n",
    "    if not y_true_val is None:\n",
    "        # this is useful when transfer learning is used with cross validation\n",
    "        res['accuracy_val'] = accuracy_score(y_true_val, y_pred_val)\n",
    "\n",
    "    res['recall'] = recall_score(y_true, y_pred, average='macro')\n",
    "    res['duration'] = duration\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_test_duration(file_name, test_duration):\n",
    "    res = pd.DataFrame(data=np.zeros((1, 1), dtype=np.float), index=[0],\n",
    "                       columns=['test_duration'])\n",
    "    res['test_duration'] = test_duration\n",
    "    res.to_csv(file_name, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_results_csv(output_file_name, root_dir):\n",
    "    res = pd.DataFrame(data=np.zeros((0, 7), dtype=np.float), index=[],\n",
    "                       columns=['classifier_name', 'archive_name', 'dataset_name',\n",
    "                                'precision', 'accuracy', 'recall', 'duration'])\n",
    "    for classifier_name in CLASSIFIERS:\n",
    "        for archive_name in ARCHIVE_NAMES:\n",
    "            datasets_dict = read_all_datasets(root_dir, archive_name)\n",
    "            for it in range(ITERATIONS):\n",
    "                curr_archive_name = archive_name\n",
    "                if it != 0:\n",
    "                    curr_archive_name = curr_archive_name + '_itr_' + str(it)\n",
    "                for dataset_name in datasets_dict.keys():\n",
    "                    output_dir = root_dir + '/results/' + classifier_name + '/' \\\n",
    "                                 + curr_archive_name + '/' + dataset_name + '/' + 'df_metrics.csv'\n",
    "                    if not os.path.exists(output_dir):\n",
    "                        continue\n",
    "                    df_metrics = pd.read_csv(output_dir)\n",
    "                    df_metrics['classifier_name'] = classifier_name\n",
    "                    df_metrics['archive_name'] = archive_name\n",
    "                    df_metrics['dataset_name'] = dataset_name\n",
    "                    res = pd.concat((res, df_metrics), axis=0, sort=False)\n",
    "\n",
    "    res.to_csv(root_dir + output_file_name, index=False)\n",
    "    # aggreagte the accuracy for iterations on same dataset\n",
    "    res = pd.DataFrame({\n",
    "        'accuracy': res.groupby(\n",
    "            ['classifier_name', 'archive_name', 'dataset_name'])['accuracy'].mean()\n",
    "    }).reset_index()\n",
    "\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_epochs_metric(hist, file_name, metric='loss'):\n",
    "    plt.figure()\n",
    "    plt.plot(hist.history[metric])\n",
    "    plt.plot(hist.history['val_' + metric])\n",
    "    plt.title('model ' + metric)\n",
    "    plt.ylabel(metric, fontsize='large')\n",
    "    plt.xlabel('epoch', fontsize='large')\n",
    "    plt.legend(['train', 'val'], loc='upper left')\n",
    "    plt.savefig(file_name, bbox_inches='tight')\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_logs_t_leNet(output_directory, hist, y_pred, y_true, duration):\n",
    "    hist_df = pd.DataFrame(hist.history)\n",
    "    hist_df.to_csv(output_directory + 'history.csv', index=False)\n",
    "\n",
    "    df_metrics = calculate_metrics(y_true, y_pred, duration)\n",
    "    df_metrics.to_csv(output_directory + 'df_metrics.csv', index=False)\n",
    "\n",
    "    index_best_model = hist_df['loss'].idxmin()\n",
    "    row_best_model = hist_df.loc[index_best_model]\n",
    "\n",
    "    df_best_model = pd.DataFrame(data=np.zeros((1, 6), dtype=np.float), index=[0],\n",
    "                                 columns=['best_model_train_loss', 'best_model_val_loss', 'best_model_train_acc',\n",
    "                                          'best_model_val_acc', 'best_model_learning_rate', 'best_model_nb_epoch'])\n",
    "\n",
    "    df_best_model['best_model_train_loss'] = row_best_model['loss']\n",
    "    df_best_model['best_model_val_loss'] = row_best_model['val_loss']\n",
    "    df_best_model['best_model_train_acc'] = row_best_model['acc']\n",
    "    df_best_model['best_model_val_acc'] = row_best_model['val_acc']\n",
    "    df_best_model['best_model_nb_epoch'] = index_best_model\n",
    "\n",
    "    df_best_model.to_csv(output_directory + 'df_best_model.csv', index=False)\n",
    "\n",
    "    # plot losses\n",
    "    plot_epochs_metric(hist, output_directory + 'epochs_loss.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_logs(output_directory, hist, y_pred, y_true, duration, lr=True, y_true_val=None, y_pred_val=None):\n",
    "    hist_df = pd.DataFrame(hist.history)\n",
    "    hist_df.to_csv(output_directory + 'history.csv', index=False)\n",
    "\n",
    "    df_metrics = calculate_metrics(y_true, y_pred, duration, y_true_val, y_pred_val)\n",
    "    df_metrics.to_csv(output_directory + 'df_metrics.csv', index=False)\n",
    "\n",
    "    index_best_model = hist_df['loss'].idxmin()\n",
    "    row_best_model = hist_df.loc[index_best_model]\n",
    "\n",
    "    df_best_model = pd.DataFrame(data=np.zeros((1, 6), dtype=np.float), index=[0],\n",
    "                                 columns=['best_model_train_loss', 'best_model_val_loss', 'best_model_train_acc',\n",
    "                                          'best_model_val_acc', 'best_model_learning_rate', 'best_model_nb_epoch'])\n",
    "\n",
    "    df_best_model['best_model_train_loss'] = row_best_model['loss']\n",
    "    df_best_model['best_model_val_loss'] = row_best_model['val_loss']\n",
    "    df_best_model['best_model_train_acc'] = row_best_model['accuracy']\n",
    "    df_best_model['best_model_val_acc'] = row_best_model['val_accuracy']\n",
    "    df_best_model['best_model_nb_epoch'] = index_best_model\n",
    "\n",
    "    df_best_model.to_csv(output_directory + 'df_best_model.csv', index=False)\n",
    "\n",
    "    # for FCN there is no hyperparameters fine tuning - everything is static in code\n",
    "\n",
    "    # plot losses\n",
    "    plot_epochs_metric(hist, output_directory + 'epochs_loss.png')\n",
    "\n",
    "    return df_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_filter(root_dir):\n",
    "    import tensorflow.keras as keras\n",
    "    classifier = 'resnet'\n",
    "    archive_name = 'UCRArchive_2018'\n",
    "    dataset_name = 'GunPoint'\n",
    "    datasets_dict = read_dataset(root_dir, archive_name, dataset_name)\n",
    "\n",
    "    x_train = datasets_dict[dataset_name][0]\n",
    "    y_train = datasets_dict[dataset_name][1]\n",
    "\n",
    "    x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], 1)\n",
    "\n",
    "    model = keras.models.load_model(\n",
    "        root_dir + 'results/' + classifier + '/' + archive_name + '/' + dataset_name + '/best_model.hdf5')\n",
    "\n",
    "    # filters\n",
    "    filters = model.layers[1].get_weights()[0]\n",
    "\n",
    "    new_input_layer = model.inputs\n",
    "    new_output_layer = [model.layers[1].output]\n",
    "\n",
    "    new_feed_forward = keras.backend.function(new_input_layer, new_output_layer)\n",
    "\n",
    "    classes = np.unique(y_train)\n",
    "\n",
    "    colors = [(255 / 255, 160 / 255, 14 / 255), (181 / 255, 87 / 255, 181 / 255)]\n",
    "    colors_conv = [(210 / 255, 0 / 255, 0 / 255), (27 / 255, 32 / 255, 101 / 255)]\n",
    "\n",
    "    idx = 10\n",
    "    idx_filter = 1\n",
    "\n",
    "    filter = filters[:, 0, idx_filter]\n",
    "\n",
    "    plt.figure(1)\n",
    "    plt.plot(filter + 0.5, color='gray', label='filter')\n",
    "    for c in classes:\n",
    "        c_x_train = x_train[np.where(y_train == c)]\n",
    "        convolved_filter_1 = new_feed_forward([c_x_train])[0]\n",
    "\n",
    "        idx_c = int(c) - 1\n",
    "\n",
    "        plt.plot(c_x_train[idx], color=colors[idx_c], label='class' + str(idx_c) + '-raw')\n",
    "        plt.plot(convolved_filter_1[idx, :, idx_filter], color=colors_conv[idx_c], label='class' + str(idx_c) + '-conv')\n",
    "        plt.legend()\n",
    "\n",
    "    plt.savefig(root_dir + 'convolution-' + dataset_name + '.pdf')\n",
    "\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_perf_themes(root_dir, df):\n",
    "    df_themes = df.copy()\n",
    "    themes_index = []\n",
    "    # add the themes\n",
    "    for dataset_name in df.index:\n",
    "        themes_index.append(utils.constants.dataset_types[dataset_name])\n",
    "\n",
    "    themes_index = np.array(themes_index)\n",
    "    themes, themes_counts = np.unique(themes_index, return_counts=True)\n",
    "    df_themes.index = themes_index\n",
    "    df_themes = df_themes.rank(axis=1, method='min', ascending=False)\n",
    "    df_themes = df_themes.where(df_themes.values == 1)\n",
    "    df_themes = df_themes.groupby(level=0).sum(axis=1)\n",
    "    df_themes['#'] = themes_counts\n",
    "\n",
    "    for classifier in CLASSIFIERS:\n",
    "        df_themes[classifier] = df_themes[classifier] / df_themes['#'] * 100\n",
    "    df_themes = df_themes.round(decimals=1)\n",
    "    df_themes.to_csv(root_dir + 'tab-perf-theme.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_perf_train_size(root_dir, df):\n",
    "    df_size = df.copy()\n",
    "    train_sizes = []\n",
    "    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')\n",
    "    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')\n",
    "    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)\n",
    "\n",
    "    for dataset_name in df.index:\n",
    "        train_size = len(datasets_dict[dataset_name][0])\n",
    "        train_sizes.append(train_size)\n",
    "\n",
    "    train_sizes = np.array(train_sizes)\n",
    "    bins = np.array([0, 100, 400, 800, 99999])\n",
    "    train_size_index = np.digitize(train_sizes, bins)\n",
    "    train_size_index = bins[train_size_index]\n",
    "\n",
    "    df_size.index = train_size_index\n",
    "    df_size = df_size.rank(axis=1, method='min', ascending=False)\n",
    "    df_size = df_size.groupby(level=0, axis=0).mean()\n",
    "    df_size = df_size.round(decimals=2)\n",
    "\n",
    "    print(df_size.to_string())\n",
    "    df_size.to_csv(root_dir + 'tab-perf-train-size.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_perf_classes(root_dir, df):\n",
    "    df_classes = df.copy()\n",
    "    class_numbers = []\n",
    "    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')\n",
    "    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')\n",
    "    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)\n",
    "\n",
    "    for dataset_name in df.index:\n",
    "        train_size = len(np.unique(datasets_dict[dataset_name][1]))\n",
    "        class_numbers.append(train_size)\n",
    "\n",
    "    class_numbers = np.array(class_numbers)\n",
    "    bins = np.array([0, 3, 4, 6, 8, 13, 9999])\n",
    "    class_numbers_index = np.digitize(class_numbers, bins)\n",
    "    class_numbers_index = bins[class_numbers_index]\n",
    "\n",
    "    df_classes.index = class_numbers_index\n",
    "    df_classes = df_classes.rank(axis=1, method='min', ascending=False)\n",
    "    df_classes = df_classes.groupby(level=0, axis=0).mean()\n",
    "    df_classes = df_classes.round(decimals=2)\n",
    "\n",
    "    print(df_classes.to_string())\n",
    "    df_classes.to_csv(root_dir + 'tab-perf-classes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def viz_perf_length(root_dir, df):\n",
    "    df_lengths = df.copy()\n",
    "    lengths = []\n",
    "    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')\n",
    "    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')\n",
    "    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)\n",
    "\n",
    "    for dataset_name in df.index:\n",
    "        length = datasets_dict[dataset_name][0].shape[1]\n",
    "        lengths.append(length)\n",
    "\n",
    "    lengths = np.array(lengths)\n",
    "    bins = np.array([0, 81, 251, 451, 700, 1001, 9999])\n",
    "    lengths_index = np.digitize(lengths, bins)\n",
    "    lengths_index = bins[lengths_index]\n",
    "\n",
    "    df_lengths.index = lengths_index\n",
    "    df_lengths = df_lengths.rank(axis=1, method='min', ascending=False)\n",
    "    df_lengths = df_lengths.groupby(level=0, axis=0).mean()\n",
    "    df_lengths = df_lengths.round(decimals=2)\n",
    "\n",
    "    print(df_lengths.to_string())\n",
    "    df_lengths.to_csv(root_dir + 'tab-perf-lengths.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "def viz_plot(root_dir, df):\n",
    "    df_lengths = df.copy()\n",
    "    lengths = []\n",
    "    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')\n",
    "    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')\n",
    "    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)\n",
    "\n",
    "    for dataset_name in df.index:\n",
    "        length = datasets_dict[dataset_name][0].shape[1]\n",
    "        lengths.append(length)\n",
    "\n",
    "    lengths_index = np.array(lengths)\n",
    "\n",
    "    df_lengths.index = lengths_index\n",
    "\n",
    "    plt.scatter(x=df_lengths['fcn'], y=df_lengths['resnet'])\n",
    "    plt.ylim(ymin=0, ymax=1.05)\n",
    "    plt.xlim(xmin=0, xmax=1.05)\n",
    "    # df_lengths['fcn']\n",
    "    plt.savefig(root_dir + 'plot.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 1
   },
   "outputs": [],
   "source": [
    "\"\"\"## Calling Models\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_ecg1_classifier(classifier_name, output_directory):\n",
    "\n",
    "    # Download the dataset\n",
    "    dataframe = pd.read_csv('http://storage.googleapis.com/download.tensorflow.org/data/ecg.csv', header=None)\n",
    "    raw_data = dataframe.values\n",
    "\n",
    "    # The last element contains the labels\n",
    "    labels = raw_data[:, -1]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    data = raw_data[:, 0:-1]\n",
    "\n",
    "    # Split the data further into training, validation, and test datasets\n",
    "    test_size = 0.2\n",
    "    randomseed = 12345\n",
    "\n",
    "    x_train, x_test, y_train, y_test = train_test_split(data, labels, test_size=test_size, random_state=randomseed)\n",
    "\n",
    "    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))\n",
    "\n",
    "    # transform the labels from integers to one hot vectors\n",
    "    enc = OneHotEncoder(categories='auto')\n",
    "    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))\n",
    "    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()\n",
    "    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()\n",
    "\n",
    "    # save orignal y because later we will use binary\n",
    "    y_true = np.argmax(y_test, axis=1)\n",
    "\n",
    "    if len(x_train.shape) == 2:  # if univariate\n",
    "        # add a dimension to make it multivariate with one dimension \n",
    "        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))\n",
    "        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))\n",
    "\n",
    "    input_shape = x_train.shape[1:]\n",
    "    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)\n",
    "\n",
    "    classifier.fit(x_train, y_train, x_test, y_test, y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_ecg200_classifier(classifier_name, output_directory):\n",
    "\n",
    "        # Download the dataset\n",
    "    train = pd.read_csv('datasets/ecg200/ECG200_TRAIN.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = train.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "\n",
    "    train.head()\n",
    "    train.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_train = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_train = raw_data[:, 1:]\n",
    "\n",
    "    test = pd.read_csv('datasets/ecg200/ECG200_TEST.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = test.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "    #dataframe.head()\n",
    "    test.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_test = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_test = raw_data[:, 1:]\n",
    "\n",
    "    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))\n",
    "\n",
    "    # transform the labels from integers to one hot vectors\n",
    "    enc = OneHotEncoder(categories='auto')\n",
    "    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))\n",
    "    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()\n",
    "    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()\n",
    "\n",
    "    # save orignal y because later we will use binary\n",
    "    y_true = np.argmax(y_test, axis=1)\n",
    "\n",
    "    if len(x_train.shape) == 2:  # if univariate\n",
    "        # add a dimension to make it multivariate with one dimension \n",
    "        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))\n",
    "        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))\n",
    "\n",
    "    input_shape = x_train.shape[1:]\n",
    "    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)\n",
    "\n",
    "    classifier.fit(x_train, y_train, x_test, y_test, y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_ecg5000_classifier(classifier_name, output_directory):\n",
    "\n",
    "        # Download the dataset\n",
    "    train = pd.read_csv('datasets/ecg5000/ECG5000_TRAIN.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = train.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "\n",
    "    train.head()\n",
    "    train.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_train = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_train = raw_data[:, 1:]\n",
    "\n",
    "    test = pd.read_csv('datasets/ecg5000/ECG5000_TEST.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = test.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "    #dataframe.head()\n",
    "    test.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_test = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_test = raw_data[:, 1:]\n",
    "\n",
    "    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))\n",
    "\n",
    "    # transform the labels from integers to one hot vectors\n",
    "    enc = OneHotEncoder(categories='auto')\n",
    "    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))\n",
    "    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()\n",
    "    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()\n",
    "\n",
    "    # save orignal y because later we will use binary\n",
    "    y_true = np.argmax(y_test, axis=1)\n",
    "\n",
    "    if len(x_train.shape) == 2:  # if univariate\n",
    "        # add a dimension to make it multivariate with one dimension \n",
    "        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))\n",
    "        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))\n",
    "\n",
    "    input_shape = x_train.shape[1:]\n",
    "    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)\n",
    "\n",
    "    classifier.fit(x_train, y_train, x_test, y_test, y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_ecgfivedays_classifier(classifier_name, output_directory):\n",
    "\n",
    "        # Download the dataset\n",
    "    train = pd.read_csv('datasets/ecgfivedays/ECGFiveDays_TRAIN.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = train.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "\n",
    "    train.head()\n",
    "    train.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_train = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_train = raw_data[:, 1:]\n",
    "\n",
    "    test = pd.read_csv('datasets/ecgfivedays/ECGFiveDays_TEST.txt', delimiter='  ', header=None)\n",
    "\n",
    "    numeric = test.apply(pd.to_numeric, args=('coerce',))\n",
    "\n",
    "\n",
    "    raw_data = numeric.values\n",
    "    #dataframe.head()\n",
    "    test.describe()\n",
    "\n",
    "    # The last element contains the labels\n",
    "    y_test = raw_data[:, 0]\n",
    "\n",
    "    # The other data points are the electrocadriogram data\n",
    "    x_test = raw_data[:, 1:]\n",
    "\n",
    "    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))\n",
    "\n",
    "    # transform the labels from integers to one hot vectors\n",
    "    enc = OneHotEncoder(categories='auto')\n",
    "    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))\n",
    "    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()\n",
    "    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()\n",
    "\n",
    "    # save orignal y because later we will use binary\n",
    "    y_true = np.argmax(y_test, axis=1)\n",
    "\n",
    "    if len(x_train.shape) == 2:  # if univariate\n",
    "        # add a dimension to make it multivariate with one dimension \n",
    "        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))\n",
    "        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))\n",
    "\n",
    "    input_shape = x_train.shape[1:]\n",
    "    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)\n",
    "\n",
    "    classifier.fit(x_train, y_train, x_test, y_test, y_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_classifier(classifier_name, input_shape, nb_classes, output_directory, verbose=True):\n",
    "\n",
    "    if classifier_name == 'mlp':\n",
    "        return Classifier_MLP(output_directory, input_shape, nb_classes, verbose)\n",
    "    if classifier_name == 'resnet':\n",
    "        return Classifier_RESNET(output_directory, input_shape, nb_classes, verbose)\n",
    "    if classifier_name == 'mcnn':\n",
    "        return Classifier_MCNN(output_directory, verbose)\n",
    "    if classifier_name == 'tlenet':\n",
    "        return Classifier_TLENET(output_directory, verbose)\n",
    "    if classifier_name == 'encoder':\n",
    "        return Classifier_ENCODER(output_directory, input_shape, nb_classes, verbose)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"\"\"### ECG 1 - MLP\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fit_ecg1_classifier('mlp', 'results/ecg1/mlp/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"\"\"### RESNET\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fit_ecg1_classifier('resnet', 'results/ecg1/resnet/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "\"\"\"### MCNN\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\"\"\"### t-lenet\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "fit_ecg1_classifier('encoder', 'results/ecg1/encoder/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# But its actually 3, because 0\n",
    "iterations = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "models = ['mlp', 'resnet', 'mcnn', 'tlenet', 'encoder']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "for i in range(iterations):\n",
    "\n",
    "    # iterate over each model\n",
    "    for model in models:\n",
    "\n",
    "         fit_ecg1_classifier(model, 'results' + str(i) + '/ecg1/' + model + '/')\n",
    "\n",
    "    for model in models:\n",
    "\n",
    "        fit_ecg200_classifier(model, 'results' + str(i) + '/ecg200/' + model + '/')\n",
    "\n",
    "    for model in models:\n",
    "\n",
    "        fit_ecg5000_classifier(model, 'results' + str(i) + '/ecg5000/' + model + '/')\n",
    "\n",
    "    for model in models:\n",
    "\n",
    "        fit_ecgfivedays_classifier(model, 'results' + str(i) + '/ecgfivedays/' + model + '/')"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "-all",
   "encoding": "# -*- coding: utf-8 -*-",
   "main_language": "python",
   "notebook_metadata_filter": "-all"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}