scc413_cw1.py

# -*- coding: utf-8 -*-
"""SCC413 Coursework1.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1v7CoumwRlu7_Ose6HMNrlDpr2oCqtn3q

#SCC413 Coursework 1

### Imports
"""

import sys
import os
import operator
import time

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_addons as tfa
import tensorflow.keras as keras

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.metrics import accuracy_score, precision_score, recall_score
from sklearn.model_selection import train_test_split

from scipy.interpolate import interp1d
from scipy.io import loadmat

"""### Test on ECG Data

The dataset you will use is based on one from [timeseriesclassification.com](http://www.timeseriesclassification.com/description.php?Dataset=ECG5000).

Try to design and train your MLP to classify normal and abnormal ECG samples.
"""


"""# Model Object Definitions

## MLP Model
"""

class Classifier_MLP:

	def __init__(self, output_directory, input_shape, nb_classes, verbose=False,build=True):
		self.output_directory = output_directory
		if build == True:
			self.model = self.build_model(input_shape, nb_classes)
			if(verbose==True):
				self.model.summary()
			self.verbose = verbose
			self.model.save_weights(self.output_directory + 'model_init.hdf5')
		return

	def build_model(self, input_shape, nb_classes):
		input_layer = keras.layers.Input(input_shape)

		# flatten/reshape because when multivariate all should be on the same axis 
		input_layer_flattened = keras.layers.Flatten()(input_layer)
		
		layer_1 = keras.layers.Dropout(0.1)(input_layer_flattened)
		layer_1 = keras.layers.Dense(500, activation='relu')(layer_1)

		layer_2 = keras.layers.Dropout(0.2)(layer_1)
		layer_2 = keras.layers.Dense(500, activation='relu')(layer_2)

		layer_3 = keras.layers.Dropout(0.2)(layer_2)
		layer_3 = keras.layers.Dense(500, activation='relu')(layer_3)

		output_layer = keras.layers.Dropout(0.3)(layer_3)
		output_layer = keras.layers.Dense(nb_classes, activation='softmax')(output_layer)

		model = keras.models.Model(inputs=input_layer, outputs=output_layer)

		model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adadelta(),
			metrics=['accuracy'])

		reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=200, min_lr=0.1)

		file_path = self.output_directory+'best_model.hdf5' 

		model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', 
			save_best_only=True)

		self.callbacks = [reduce_lr,model_checkpoint]

		return model

	def fit(self, x_train, y_train, x_val, y_val,y_true):
		if not tf.test.is_gpu_available:
			print('error')
			exit()
		# x_val and y_val are only used to monitor the test loss and NOT for training  
		batch_size = 16
		nb_epochs = 500

		mini_batch_size = int(min(x_train.shape[0]/10, batch_size))

		start_time = time.time() 

		hist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,
			verbose=self.verbose, validation_data=(x_val,y_val), callbacks=self.callbacks)
		
		duration = time.time() - start_time

		self.model.save(self.output_directory + 'last_model.hdf5')

		model = keras.models.load_model(self.output_directory+'best_model.hdf5')

		y_pred = model.predict(x_val)

		# convert the predicted from binary to integer 
		y_pred = np.argmax(y_pred , axis=1)

		save_logs(self.output_directory, hist, y_pred, y_true, duration)

		keras.backend.clear_session()

	def predict(self, x_test, y_true,x_train,y_train,y_test,return_df_metrics = True):
		model_path = self.output_directory + 'best_model.hdf5'
		model = keras.models.load_model(model_path)
		y_pred = model.predict(x_test)
		if return_df_metrics:
			y_pred = np.argmax(y_pred, axis=1)
			df_metrics = calculate_metrics(y_true, y_pred, 0.0)
			return df_metrics
		else:
			return y_pred

"""## RESNET Model"""

class Classifier_RESNET:

    def __init__(self, output_directory, input_shape, nb_classes, verbose=False, build=True, load_weights=False):
        self.output_directory = output_directory
        if build == True:
            self.model = self.build_model(input_shape, nb_classes)
            if (verbose == True):
                self.model.summary()
            self.verbose = verbose
            if load_weights == True:
                self.model.load_weights(self.output_directory
                                        .replace('resnet_augment', 'resnet')
                                        .replace('TSC_itr_augment_x_10', 'TSC_itr_10')
                                        + '/model_init.hdf5')
            else:
                self.model.save_weights(self.output_directory + 'model_init.hdf5')
        return

    def build_model(self, input_shape, nb_classes):
        n_feature_maps = 64

        input_layer = keras.layers.Input(input_shape)

        # BLOCK 1

        conv_x = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=8, padding='same')(input_layer)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation('relu')(conv_x)

        conv_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=5, padding='same')(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation('relu')(conv_y)

        conv_z = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=3, padding='same')(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # expand channels for the sum
        shortcut_y = keras.layers.Conv1D(filters=n_feature_maps, kernel_size=1, padding='same')(input_layer)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)

        output_block_1 = keras.layers.add([shortcut_y, conv_z])
        output_block_1 = keras.layers.Activation('relu')(output_block_1)

        # BLOCK 2

        conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_1)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation('relu')(conv_x)

        conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation('relu')(conv_y)

        conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # expand channels for the sum
        shortcut_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=1, padding='same')(output_block_1)
        shortcut_y = keras.layers.BatchNormalization()(shortcut_y)

        output_block_2 = keras.layers.add([shortcut_y, conv_z])
        output_block_2 = keras.layers.Activation('relu')(output_block_2)

        # BLOCK 3

        conv_x = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=8, padding='same')(output_block_2)
        conv_x = keras.layers.BatchNormalization()(conv_x)
        conv_x = keras.layers.Activation('relu')(conv_x)

        conv_y = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=5, padding='same')(conv_x)
        conv_y = keras.layers.BatchNormalization()(conv_y)
        conv_y = keras.layers.Activation('relu')(conv_y)

        conv_z = keras.layers.Conv1D(filters=n_feature_maps * 2, kernel_size=3, padding='same')(conv_y)
        conv_z = keras.layers.BatchNormalization()(conv_z)

        # no need to expand channels because they are equal
        shortcut_y = keras.layers.BatchNormalization()(output_block_2)

        output_block_3 = keras.layers.add([shortcut_y, conv_z])
        output_block_3 = keras.layers.Activation('relu')(output_block_3)

        # FINAL

        gap_layer = keras.layers.GlobalAveragePooling1D()(output_block_3)

        output_layer = keras.layers.Dense(nb_classes, activation='softmax')(gap_layer)

        model = keras.models.Model(inputs=input_layer, outputs=output_layer)

        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(),
                      metrics=['accuracy'])

        reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='loss', factor=0.5, patience=50, min_lr=0.0001)

        file_path = self.output_directory + 'best_model.hdf5'

        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss',
                                                           save_best_only=True)

        self.callbacks = [reduce_lr, model_checkpoint]

        return model

    def fit(self, x_train, y_train, x_val, y_val, y_true):
        if not tf.test.is_gpu_available:
            print('error')
            exit()
        # x_val and y_val are only used to monitor the test loss and NOT for training
        batch_size = 64
        nb_epochs = 500

        mini_batch_size = int(min(x_train.shape[0] / 10, batch_size))

        start_time = time.time()

        hist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,
                              verbose=self.verbose, validation_data=(x_val, y_val), callbacks=self.callbacks)

        duration = time.time() - start_time

        self.model.save(self.output_directory + 'last_model.hdf5')

        y_pred = self.predict(x_val, y_true, x_train, y_train, y_val,
                              return_df_metrics=False)

        # save predictions
        np.save(self.output_directory + 'y_pred.npy', y_pred)

        # convert the predicted from binary to integer
        y_pred = np.argmax(y_pred, axis=1)

        df_metrics = save_logs(self.output_directory, hist, y_pred, y_true, duration)

        keras.backend.clear_session()

        return df_metrics

    def predict(self, x_test, y_true, x_train, y_train, y_test, return_df_metrics=True):
        start_time = time.time()
        model_path = self.output_directory + 'best_model.hdf5'
        model = keras.models.load_model(model_path)
        y_pred = model.predict(x_test)
        if return_df_metrics:
            y_pred = np.argmax(y_pred, axis=1)
            df_metrics = calculate_metrics(y_true, y_pred, 0.0)
            return df_metrics
        else:
            test_duration = time.time() - start_time
            save_test_duration(self.output_directory + 'test_duration.csv', test_duration)
            return y_pred

"""### Encoder """

class Classifier_ENCODER:

    def __init__(self, output_directory, input_shape, nb_classes, verbose=False,build=True):
        self.output_directory = output_directory
        if build == True:
            self.model = self.build_model(input_shape, nb_classes)
            if (verbose == True):
                self.model.summary()
            self.verbose = verbose
            self.model.save_weights(self.output_directory + 'model_init.hdf5')

    def build_model(self, input_shape, nb_classes):
        input_layer = keras.layers.Input(input_shape)

        # conv block -1
        conv1 = keras.layers.Conv1D(filters=128,kernel_size=5,strides=1,padding='same')(input_layer)
        conv1 = tfa.layers.InstanceNormalization()(conv1)
        conv1 = keras.layers.PReLU(shared_axes=[1])(conv1)
        conv1 = keras.layers.Dropout(rate=0.2)(conv1)
        conv1 = keras.layers.MaxPooling1D(pool_size=2)(conv1)
        # conv block -2
        conv2 = keras.layers.Conv1D(filters=256,kernel_size=11,strides=1,padding='same')(conv1)
        conv2 = tfa.layers.InstanceNormalization()(conv2)
        conv2 = keras.layers.PReLU(shared_axes=[1])(conv2)
        conv2 = keras.layers.Dropout(rate=0.2)(conv2)
        conv2 = keras.layers.MaxPooling1D(pool_size=2)(conv2)
        # conv block -3
        conv3 = keras.layers.Conv1D(filters=512,kernel_size=21,strides=1,padding='same')(conv2)
        conv3 = tfa.layers.InstanceNormalization()(conv3)
        conv3 = keras.layers.PReLU(shared_axes=[1])(conv3)
        conv3 = keras.layers.Dropout(rate=0.2)(conv3)
        # split for attention
        attention_data = keras.layers.Lambda(lambda x: x[:,:,:256])(conv3)
        attention_softmax = keras.layers.Lambda(lambda x: x[:,:,256:])(conv3)
        # attention mechanism
        attention_softmax = keras.layers.Softmax()(attention_softmax)
        multiply_layer = keras.layers.Multiply()([attention_softmax,attention_data])
        # last layer
        dense_layer = keras.layers.Dense(units=256,activation='sigmoid')(multiply_layer)
        dense_layer = tfa.layers.InstanceNormalization()(dense_layer)
        # output layer
        flatten_layer = keras.layers.Flatten()(dense_layer)
        output_layer = keras.layers.Dense(units=nb_classes,activation='softmax')(flatten_layer)

        model = keras.models.Model(inputs=input_layer, outputs=output_layer)

        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(0.00001),
                      metrics=['accuracy'])

        file_path = self.output_directory + 'best_model.hdf5'

        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path,
                                                           monitor='loss', save_best_only=True)

        self.callbacks = [model_checkpoint]

        return model

    def fit(self, x_train, y_train, x_val, y_val, y_true):
        if not tf.test.is_gpu_available:
            print('error')
            exit()
        # x_val and y_val are only used to monitor the test loss and NOT for training
        batch_size = 12
        nb_epochs = 100

        mini_batch_size = batch_size

        start_time = time.time()

        hist = self.model.fit(x_train, y_train, batch_size=mini_batch_size, epochs=nb_epochs,
                              verbose=self.verbose, validation_data=(x_val, y_val), callbacks=self.callbacks)

        duration = time.time() - start_time

        self.model.save(self.output_directory+'last_model.hdf5')

        model = keras.models.load_model(self.output_directory + 'best_model.hdf5')

        y_pred = model.predict(x_val)

        # convert the predicted from binary to integer
        y_pred = np.argmax(y_pred, axis=1)

        save_logs(self.output_directory, hist, y_pred, y_true, duration, lr=False)

        keras.backend.clear_session()

    def predict(self, x_test,y_true,x_train,y_train,y_test,return_df_metrics = True):
        model_path = self.output_directory + 'best_model.hdf5'
        model = keras.models.load_model(model_path)
        y_pred = model.predict(x_test)
        if return_df_metrics:
            y_pred = np.argmax(y_pred, axis=1)
            df_metrics = calculate_metrics(y_true, y_pred, 0.0)
            return df_metrics
        else:
            return y_pred

"""## MCNN Model"""

class Classifier_MCNN:

    def __init__(self, output_directory, verbose, build=True):
        self.output_directory = output_directory
        self.verbose = verbose
        #self.pool_factors = [2,3,5] # used for hyperparameters grid search
        self.pool_factors = [2, 3, 5] # used for hyperparameters grid search
        self.filter_sizes = [0.05,0.1,0.2] # used for hyperparameters grid search 

    def slice_data(self, data_x, data_y, slice_ratio): 
        n = data_x.shape[0]
        length = data_x.shape[1]
        n_dim = data_x.shape[2] # for MTS 
        nb_classes = data_y.shape[1]

        length_sliced = int(length * slice_ratio)
       
        increase_num = length - length_sliced + 1 #if increase_num =5, it means one ori becomes 5 new instances.
        n_sliced = n * increase_num

        new_x = np.zeros((n_sliced, length_sliced,n_dim))
        new_y = np.zeros((n_sliced,nb_classes))
        for i in range(n):
            for j in range(increase_num):
                new_x[i * increase_num + j, :,:] = data_x[i,j : j + length_sliced,:]
                new_y[i * increase_num + j] = np.int_(data_y[i].astype(np.float32))

        return new_x, new_y

    def split_train(self, train_x,train_y):
        #shuffle for splitting train set and dataset
        n = train_x.shape[0]
        ind = np.arange(n)
        np.random.shuffle(ind) #shuffle the train set
        
        #split train set into train set and validation set
        valid_x = train_x[ind[0:int(0.2 * n)]]
        valid_y = train_y[ind[0:int(0.2 * n)]]

        ind = np.delete(ind, (range(0,int(0.2 * n))))

        train_x = train_x[ind] 
        train_y = train_y[ind]
            
        return train_x,train_y,valid_x,valid_y

    def _downsample(self, data_x, sample_rate, offset = 0):
        num = data_x.shape[0]
        length_x = data_x.shape[1]
        num_dim = data_x.shape[2] # for MTS 
        last_one = 0
        if length_x % sample_rate > offset:
            last_one = 1
        new_length = int(np.floor( length_x / sample_rate)) + last_one
        output = np.zeros((num, new_length,num_dim))
        for i in range(new_length):
            output[:,i] = np.array(data_x[:,offset + sample_rate * i])

        return output

    def _movingavrg(self, data_x, window_size):
        num = data_x.shape[0]
        length_x = data_x.shape[1]
        num_dim = data_x.shape[2] # for MTS 
        output_len = length_x - window_size + 1
        output = np.zeros((num, output_len,num_dim))
        for i in range(output_len):
            output[:,i] = np.mean(data_x[:, i : i + window_size], axis = 1)
        return output

    def movingavrg(self, data_x, window_base, step_size, num):
        if num == 0:
            return (None, [])
        out = self._movingavrg(data_x, window_base)
        data_lengths = [out.shape[1]]
        for i in range(1, num):
            window_size = window_base + step_size * i
            if window_size > data_x.shape[1]:
                continue
            new_series = self._movingavrg(data_x, window_size)
            data_lengths.append( new_series.shape[1] )
            out = np.concatenate([out, new_series], axis = 1)
        return (out, data_lengths)

    def batch_movingavrg(self, train,valid,test, window_base, step_size, num):
        (new_train, lengths) = self.movingavrg(train, window_base, step_size, num)
        (new_valid, lengths) = self.movingavrg(valid, window_base, step_size, num)
        (new_test, lengths) = self.movingavrg(test, window_base, step_size, num)
        return (new_train, new_valid, new_test, lengths)

    def downsample(self, data_x, base, step_size, num):
        # the case for dataset JapaneseVowels MTS
        if data_x.shape[1] ==26 :
            return (None,[]) # too short to apply downsampling
        if num == 0:
            return (None, [])
        out = self._downsample(data_x, base,0)
        data_lengths = [out.shape[1]]
        #for offset in range(1,base): #for the base case
        #    new_series = _downsample(data_x, base, offset)
        #    data_lengths.append( new_series.shape[1] )
        #    out = np.concatenate( [out, new_series], axis = 1)
        for i in range(1, num):
            sample_rate = base + step_size * i 
            if sample_rate > data_x.shape[1]:
                continue
            for offset in range(0,1):#sample_rate):
                new_series = self._downsample(data_x, sample_rate, offset)
                data_lengths.append( new_series.shape[1] )
                out = np.concatenate( [out, new_series], axis = 1)
        return (out, data_lengths)

    def batch_downsample(self, train,valid,test, window_base, step_size, num):
        (new_train, lengths) = self.downsample(train, window_base, step_size, num)
        (new_valid, lengths) = self.downsample(valid, window_base, step_size, num)
        (new_test, lengths) = self.downsample(test, window_base, step_size, num)
        return (new_train, new_valid, new_test, lengths)

    def get_pool_factor(self,conv_shape,pool_size):
        for pool_factor in self.pool_factors:
            temp_pool_size = int(int(conv_shape)/pool_factor)
            print(temp_pool_size)
            if temp_pool_size == pool_size:
                return pool_factor

        raise Exception('Error on pool factor')

    def train(self, x_train, y_train, x_test, y_test,y_true, pool_factor=None, filter_size=None,do_train=True):
        window_size = 0.2
        n_train_batch = 10
        n_epochs = 200
        max_train_batch_size = 256

        # print('Original train shape: ', x_train.shape)
        # print('Original test shape: ', x_test.shape)

        # split train into validation set with validation_size = 0.2 train_size 
        x_train,y_train,x_val,y_val = self.split_train(x_train,y_train)
        
        ori_len = x_train.shape[1] # original_length of time series  
        slice_ratio = 0.9

        if do_train == True:
            kernel_size = int(ori_len * filter_size)

        if do_train == False:
            model = keras.models.load_model(self.output_directory+'best_model.hdf5')

            # model.summary()

            pool_size = model.get_layer('max_pooling1d_1').get_config()['pool_size'][0]

            conv_shape = model.get_layer('conv1d_1').output_shape[1]


            pool_factor = self.get_pool_factor(conv_shape,pool_size)

        #restrict slice ratio when data lenght is too large
        if ori_len > 500 : 
            slice_ratio = slice_ratio if slice_ratio > 0.98 else 0.98
        elif ori_len < 16:
            slice_ratio = 0.7

        increase_num = ori_len - int(ori_len * slice_ratio) + 1 #this can be used as the bath size

        train_batch_size = int(x_train.shape[0] * increase_num / n_train_batch)
        if train_batch_size > max_train_batch_size : 
            # limit the train_batch_size 
            n_train_batch = int(x_train.shape[0] * increase_num / max_train_batch_size)
        
        # data augmentation by slicing the length of the series 
        x_train,y_train = self.slice_data(x_train,y_train,slice_ratio)
        x_val,y_val = self.slice_data(x_val,y_val,slice_ratio)
        x_test,y_test = self.slice_data(x_test,y_test,slice_ratio)

        train_set_x, train_set_y = x_train,y_train
        valid_set_x, valid_set_y = x_val,y_val
        test_set_x, _ = x_test,y_test

        valid_num = valid_set_x.shape[0]
        
        # print("increase factor is ", increase_num, ', ori len', ori_len)
        valid_num_batch = int(valid_num / increase_num)

        test_num = test_set_x.shape[0]
        test_num_batch = int(test_num / increase_num)

        length_train = train_set_x.shape[1] #length after slicing.
        
        window_size = int(length_train * window_size) if window_size < 1 else int(window_size)

        #*******set up the ma and ds********#
        ma_base,ma_step,ma_num   = 5, 6, 1
        ds_base,ds_step, ds_num  = 2, 1, 4

        ds_num_max = length_train / (pool_factor * window_size)
        ds_num = int(min(ds_num, ds_num_max))
        
        #*******set up the ma and ds********#

        (ma_train, ma_valid, ma_test , ma_lengths) = self.batch_movingavrg(train_set_x,
                                                        valid_set_x, test_set_x,
                                                        ma_base, ma_step, ma_num)
        (ds_train, ds_valid, ds_test , ds_lengths) = self.batch_downsample(train_set_x,
                                                        valid_set_x, test_set_x,
                                                        ds_base, ds_step, ds_num)

        #concatenate directly
        data_lengths = [length_train] 
        #downsample part:
        if ds_lengths != []:
            data_lengths +=  ds_lengths
            train_set_x = np.concatenate([train_set_x, ds_train], axis = 1)
            valid_set_x = np.concatenate([valid_set_x, ds_valid], axis = 1)
            test_set_x = np.concatenate([test_set_x, ds_test], axis = 1)

        #moving average part
        if ma_lengths != []:
            data_lengths += ma_lengths
            train_set_x = np.concatenate([train_set_x, ma_train], axis = 1)
            valid_set_x = np.concatenate([valid_set_x, ma_valid], axis = 1)
            test_set_x = np.concatenate([test_set_x, ma_test], axis = 1)
        # print("Data length:", data_lengths)

        n_train_size = train_set_x.shape[0]
        n_valid_size = valid_set_x.shape[0]
        n_test_size = test_set_x.shape[0]
        batch_size = int(n_train_size / n_train_batch)
        n_train_batches = int(n_train_size / batch_size)
        data_dim = train_set_x.shape[1]  
        num_dim = train_set_x.shape[2] # For MTS 
        nb_classes = train_set_y.shape[1] 

        # print('train size', n_train_size, ',valid size', n_valid_size, ' test size', n_test_size)
        # print('batch size ', batch_size)
        # print('n_train_batches is ', n_train_batches)
        # print('data dim is ', data_dim)
        # print('---------------------------')

        ######################
        # BUILD ACTUAL MODEL #
        ######################
        # print('building the model...')

        input_shapes, max_length = self.get_list_of_input_shapes(data_lengths,num_dim)

        start_time = time.time()

        best_validation_loss = np.inf

        if do_train == True:

            model = self.build_model(input_shapes, nb_classes, pool_factor, kernel_size)

            if (self.verbose==True) :
                model.summary()

            # print('Training')


            # early-stopping parameters
            patience = 10000  # look as this many examples regardless
            patience_increase = 2  # wait this much longer when a new best is
                                   # found
            improvement_threshold = 0.995  # a relative improvement of this much is
                                           # considered significant
            validation_frequency = min(n_train_batches, patience / 2)
                                          # go through this many
                                          # minibatche before checking the network
                                          # on the validation set; in this case we
                                          # check every epoch
            max_before_stopping = 500

            best_iter = 0
            valid_loss = 0.

            epoch = 0
            done_looping = False
            num_no_update_epoch = 0
            epoch_avg_cost = float('inf')
            epoch_avg_err = float('inf')

            while (epoch < n_epochs) and (not done_looping):
                epoch = epoch + 1
                epoch_train_err = 0.
                epoch_cost = 0.

                num_no_update_epoch += 1
                if num_no_update_epoch == max_before_stopping:
                    break


                for minibatch_index in range(n_train_batches):

                    iteration = (epoch - 1) * n_train_batches + minibatch_index

                    x = train_set_x[minibatch_index*batch_size: (minibatch_index+1)*batch_size]
                    y = train_set_y[minibatch_index*batch_size: (minibatch_index+1)*batch_size]

                    x = self.split_input_for_model(x,input_shapes)

                    cost_ij, accuracy = model.train_on_batch(x,y)

                    train_err = 1 - accuracy

                    epoch_train_err = epoch_train_err + train_err
                    epoch_cost = epoch_cost + cost_ij


                    if (iteration + 1) % validation_frequency == 0:

                        valid_losses = []
                        for i in range(valid_num_batch):
                            x = valid_set_x[i * (increase_num) : (i + 1) * (increase_num)]
                            y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))

                            # convert the predicted from binary to integer
                            y_pred = np.argmax(y_pred , axis=1)
                            label = np.argmax(valid_set_y[i * increase_num])

                            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)
                            unique_value = unique_value.tolist()

                            curr_err = 1.
                            if label in unique_value:
                                target_ind = unique_value.index(label)
                                count = count.tolist()
                                sorted_count = sorted(count)
                                if count[target_ind] == sorted_count[-1]:
                                    if len(sorted_count) > 1 and sorted_count[-1] == sorted_count[-2]:
                                        curr_err = 0.5 #tie
                                    else:
                                        curr_err = 0
                            valid_losses.append(curr_err)
                        valid_loss = sum(valid_losses) / float(len(valid_losses))

                        # print('...epoch%i,valid err: %.5f |' % (epoch,valid_loss))

                        # if we got the best validation score until now
                        if valid_loss <= best_validation_loss:
                            num_no_update_epoch = 0

                            #improve patience if loss improvement is good enough
                            if valid_loss < best_validation_loss*improvement_threshold:
                                patience = max(patience,iteration*patience_increase)

                            # save best validation score and iteration number
                            best_validation_loss = valid_loss
                            best_iter = iteration

                            # save model in h5 format
                            model.save(self.output_directory+'best_model.hdf5')

                        model.save(self.output_directory + 'last_model.hdf5')
                    if patience<= iteration:
                        done_looping=True
                        break
                epoch_avg_cost = epoch_cost/n_train_batches
                epoch_avg_err = epoch_train_err/n_train_batches

                # print ('train err %.5f, cost %.4f' %(epoch_avg_err,epoch_avg_cost))
                if epoch_avg_cost == 0:
                    break

            # print('Optimization complete.')

        # test the model
        # print('Testing')
        # load best model
        model = keras.models.load_model(self.output_directory+'best_model.hdf5')

        # get the true predictions of the test set
        y_predicted = []
        for i in range(test_num_batch):
            x = test_set_x[i * (increase_num) : (i + 1) * (increase_num)]
            y_pred = model.predict_on_batch(self.split_input_for_model(x,input_shapes))

            # convert the predicted from binary to integer 
            y_pred = np.argmax(y_pred , axis=1)

            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)

            idx_max = np.argmax(count)
            predicted_label = unique_value[idx_max]

            y_predicted.append(predicted_label)

        y_pred = np.array(y_predicted)

        duration = time.time() - start_time        

        df_metrics = calculate_metrics(y_true,y_pred, duration)

        # print(y_true.shape)
        # print(y_pred.shape)

        df_metrics.to_csv(self.output_directory+'df_metrics.csv', index=False)

        return df_metrics, model , best_validation_loss


    def split_input_for_model(self, x, input_shapes):
        res = []
        indx = 0 
        for input_shape in input_shapes:
            res.append(x[:,indx:indx+input_shape[0],:])
            indx = indx + input_shape[0]
        return res

    def build_model(self, input_shapes, nb_classes, pool_factor, kernel_size):
        input_layers = []
        stage_1_layers = []

        for input_shape in input_shapes: 

            input_layer = keras.layers.Input(input_shape)
            
            input_layers.append(input_layer)

            conv_layer = keras.layers.Conv1D(filters=256, kernel_size=kernel_size, padding='same', 
                activation='sigmoid', kernel_initializer='glorot_uniform')(input_layer)

            # should all concatenated have the same length
            pool_size = int(int(conv_layer.shape[1])/pool_factor)

            max_layer = keras.layers.MaxPooling1D(pool_size=pool_size)(conv_layer)

            # max_layer = keras.layers.GlobalMaxPooling1D()(conv_layer)

            stage_1_layers.append(max_layer)

        concat_layer = keras.layers.Concatenate(axis=-1)(stage_1_layers)

        kernel_size = int(min(kernel_size, int(concat_layer.shape[1]))) # kernel shouldn't exceed the length  

        full_conv = keras.layers.Conv1D(filters=256, kernel_size=kernel_size, padding='same', 
            activation='sigmoid', kernel_initializer='glorot_uniform')(concat_layer)

        pool_size = int(int(full_conv.shape[1])/pool_factor)

        full_max = keras.layers.MaxPooling1D(pool_size=pool_size)(full_conv)

        full_max = keras.layers.Flatten()(full_max) 

        fully_connected = keras.layers.Dense(units=256, activation='sigmoid', 
            kernel_initializer='glorot_uniform')(full_max)

        output_layer = keras.layers.Dense(units=nb_classes, activation='softmax', 
            kernel_initializer='glorot_uniform')(fully_connected)

        model = keras.models.Model(inputs=input_layers, outputs=output_layer)

        model.compile(loss='categorical_crossentropy', optimizer=keras.optimizers.Adam(lr=0.1),
            metrics=['accuracy'])
        
        return model 

    def get_list_of_input_shapes(self, data_lengths, num_dim):
        input_shapes = []
        max_length = 0
        for i in data_lengths: 
            input_shapes.append((i,num_dim))
            max_length = max(max_length, i)
        return input_shapes , max_length

    def fit(self, x_train, y_train, x_test, y_test,y_true):
        if not tf.test.is_gpu_available:
            print('error')
            exit()
        best_df_metrics = None
        best_valid_loss = np.inf

        output_directory_root = self.output_directory 
        # grid search 
        for pool_factor in self.pool_factors:
            for filter_size in self.filter_sizes:
                self.output_directory = output_directory_root+'/hyper_param_search/'+'/pool_factor_'+ \
                    str(pool_factor)+'/filter_size_'+str(filter_size)+'/' 
                create_directory(self.output_directory)
                df_metrics, model , valid_loss = self.train(x_train, y_train, x_test, 
                                                          y_test,y_true,pool_factor,filter_size)

                if (valid_loss < best_valid_loss): 
                    best_valid_loss = valid_loss
                    best_df_metrics = df_metrics
                    best_df_metrics.to_csv(output_directory_root+'df_metrics.csv', index=False)
                    model.save(output_directory_root+'best_model.hdf5')

                model = None
                # clear memeory 
                keras.backend.clear_session()

    def predict(self, x_test, y_true,x_train,y_train,y_test):
        df_metrics, _ , _ = self.train(x_train, y_train, x_test, y_test,y_true, do_train=False)

        return df_metrics

"""## t-LeNet Model"""

class Classifier_TLENET:
    
    def __init__(self, output_directory, verbose,build=True):
        self.output_directory = output_directory
        self.verbose = verbose
        self.warping_ratios = [0.5,1,2]
        self.slice_ratio = 0.1
        
    def slice_data(self, data_x, data_y, length_sliced): 
        n = data_x.shape[0]
        length = data_x.shape[1]
        n_dim = data_x.shape[2] # for MTS 
        nb_classes = data_y.shape[1]
       
        increase_num = length - length_sliced + 1 #if increase_num =5, it means one ori becomes 5 new instances.
        n_sliced = n * increase_num

        print((n_sliced, length_sliced,n_dim))

        new_x = np.zeros((n_sliced, length_sliced,n_dim))
        new_y = np.zeros((n_sliced,nb_classes))
        for i in range(n):
            for j in range(increase_num):
                new_x[i * increase_num + j, :,:] = data_x[i,j : j + length_sliced,:]
                new_y[i * increase_num + j] = np.int_(data_y[i].astype(np.float32))

        return new_x, new_y, increase_num 

    def window_warping(self, data_x, warping_ratio): 
        num_x = data_x.shape[0]
        len_x = data_x.shape[1]
        dim_x = data_x.shape[2]
        
        x = np.arange(0,len_x,warping_ratio)
        xp = np.arange(0,len_x)
        
        new_length = len(np.interp(x,xp,data_x[0,:,0]))
        
        warped_series = np.zeros((num_x,new_length,dim_x),dtype=np.float64)
        
        for i in range(num_x):
            for j in range(dim_x):
                warped_series[i,:,j] = np.interp(x,xp,data_x[i,:,j])
                
        return warped_series
    
    def build_model(self, input_shape, nb_classes):
        input_layer = keras.layers.Input(input_shape)
        
        conv_1 = keras.layers.Conv1D(filters=5,kernel_size=5,activation='relu', padding='same')(input_layer)
        conv_1 = keras.layers.MaxPool1D(pool_size=2)(conv_1)
        
        conv_2 = keras.layers.Conv1D(filters=20, kernel_size=5, activation='relu', padding='same')(conv_1)
        conv_2 = keras.layers.MaxPool1D(pool_size=4)(conv_2)
        
        # they did not mention the number of hidden units in the fully-connected layer
        # so we took the lenet they referenced 
        
        flatten_layer = keras.layers.Flatten()(conv_2)
        fully_connected_layer = keras.layers.Dense(500,activation='relu')(flatten_layer)
        
        output_layer = keras.layers.Dense(nb_classes,activation='softmax')(fully_connected_layer)
        
        model = keras.models.Model(inputs=input_layer,outputs=output_layer)
        
        model.compile(optimizer=keras.optimizers.Adam(lr=0.01,decay=0.005),
                      loss='categorical_crossentropy', metrics=['accuracy'])
        
        file_path = self.output_directory+'best_model.hdf5'

        model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=file_path, monitor='loss', 
            save_best_only=True)
        
        self.callbacks=[model_checkpoint]
        
        return model
    
    def pre_processing(self,x_train,y_train,x_test,y_test):
        length_ratio = int(self.slice_ratio*x_train.shape[1])
        
        x_train_augmented = [] # list of the augmented as well as the original data
        x_test_augmented = [] # list of the augmented as well as the original data
        
        y_train_augmented = []
        y_test_augmented = [] 
        
        # data augmentation using WW
        for warping_ratio in self.warping_ratios:
            x_train_augmented.append(self.window_warping(x_train,warping_ratio))
            x_test_augmented.append(self.window_warping(x_test,warping_ratio))
            y_train_augmented.append(y_train)
            y_test_augmented.append(y_test)
                
        increase_nums = []
        
        # data augmentation using WS 
        for i in range(0,len(x_train_augmented)):
            x_train_augmented[i],y_train_augmented[i],increase_num = self.slice_data(
                    x_train_augmented[i],y_train,length_ratio)
            x_test_augmented[i],y_test_augmented[i],increase_num = self.slice_data(
                    x_test_augmented[i],y_test,length_ratio)
            increase_nums.append(increase_num)
        
        tot_increase_num = np.array(increase_nums).sum()
        
        new_x_train = np.zeros((x_train.shape[0]*tot_increase_num, length_ratio,x_train.shape[2]))
        new_y_train = np.zeros((y_train.shape[0]*tot_increase_num,y_train.shape[1]))
        
        new_x_test = np.zeros((x_test.shape[0]*tot_increase_num, length_ratio,x_test.shape[2]))
        new_y_test = np.zeros((y_test.shape[0]*tot_increase_num,y_test.shape[1]))
        
        # merge the list of augmented data
        idx = 0
        for i in range(x_train.shape[0]):
            for j in range(len(increase_nums)):
                increase_num = increase_nums[j]
                new_x_train [idx:idx+increase_num ,:,:] = \
                    x_train_augmented[j][i*increase_num:(i+1)*increase_num,:,:] 
                new_y_train [idx:idx+increase_num,:] = \
                    y_train_augmented[j][i*increase_num:(i+1)*increase_num,:]
                idx += increase_num
                
        # do the same for the test set 
        idx = 0
        for i in range(x_test.shape[0]):
            for j in range(len(increase_nums)):
                increase_num = increase_nums[j]
                new_x_test [idx:idx+increase_num ,:,:] = \
                    x_test_augmented[j][i*increase_num:(i+1)*increase_num,:,:] 
                new_y_test [idx:idx+increase_num,:] = \
                    y_test_augmented[j][i*increase_num:(i+1)*increase_num,:]
                idx += increase_num
        return new_x_train,new_y_train,new_x_test,new_y_test, tot_increase_num
    
    def fit(self, x_train, y_train, x_test, y_test,y_true):
        if not tf.test.is_gpu_available:
            print('error')
            exit()
        nb_epochs = 500
        batch_size= 256
        nb_classes = y_train.shape[1]

        # limit the number of augmented time series if series too long or too many 
        if x_train.shape[1] > 500 or x_train.shape[0] > 2000 or x_test.shape[0] > 2000 : 
            self.warping_ratios = [1]
            self.slice_ratio = 0.9
        # increase the slice if series too short 
        if x_train.shape[1]*self.slice_ratio < 8:
            self.slice_ratio = 8/x_train.shape[1]

        ####################
        ## pre-processing ##
        ####################
        
        
        x_train , y_train , x_test , y_test, tot_increase_num = self.pre_processing(x_train,y_train,x_test,y_test)
        
        print('Total increased number for each MTS: ',tot_increase_num)

        #########################
        ## done pre-processing ##
        #########################
        
        input_shape = x_train.shape[1:]
        model = self.build_model(input_shape,nb_classes)
        
        if self.verbose == True:
            model.summary()
        
        start_time = time.time()
        
        hist = model.fit(x_train, y_train, batch_size=batch_size, epochs=nb_epochs,
            verbose=self.verbose, validation_data = (x_test,y_test),callbacks=self.callbacks)

        model.save(self.output_directory+'last_model.hdf5')

        model = keras.models.load_model(self.output_directory+'best_model.hdf5')
        
        y_pred = model.predict(x_test,batch_size=batch_size)
        # convert the predicted from binary to integer 
        y_pred = np.argmax(y_pred , axis=1)
        
        # get the true predictions of the test set
        y_predicted = []
        test_num_batch = int(x_test.shape[0]/tot_increase_num)
        for i in range(test_num_batch):
            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred[i*tot_increase_num:(i+1)*tot_increase_num], True, True, True)

            idx_max = np.argmax(count)
            predicted_label = unique_value[idx_max]

            y_predicted.append(predicted_label)

        y_pred = np.array(y_predicted)
        
        duration = time.time() - start_time
        
        save_logs(self.output_directory, hist, y_pred, y_true, duration )

        keras.backend.clear_session()

    def predict(self, x_test, y_true,x_train,y_train,y_test):
        batch_size = 256

        # limit the number of augmented time series if series too long or too many
        if x_train.shape[1] > 500 or x_train.shape[0] > 2000 or x_test.shape[0] > 2000:
            self.warping_ratios = [1]
            self.slice_ratio = 0.9
        # increase the slice if series too short
        if x_train.shape[1] * self.slice_ratio < 8:
            self.slice_ratio = 8 / x_train.shape[1]

        new_x_train, new_y_train, new_x_test, new_y_test, tot_increase_num = \
            self.pre_processing(x_train, y_train, x_test, y_test)

        model_path = self.output_directory + 'best_model.hdf5'
        model = keras.models.load_model(model_path)

        y_pred = model.predict(new_x_test, batch_size=batch_size)
        # convert the predicted from binary to integer
        y_pred = np.argmax(y_pred, axis=1)

        # get the true predictions of the test set
        y_predicted = []
        test_num_batch = int(new_x_test.shape[0] / tot_increase_num)
        for i in range(test_num_batch):
            unique_value, sub_ind, correspond_ind, count = np.unique(y_pred, True, True, True)

            idx_max = np.argmax(count)
            predicted_label = unique_value[idx_max]

            y_predicted.append(predicted_label)

        y_pred = np.array(y_predicted)

        df_metrics = calculate_metrics(y_true, y_pred, 0.0)
        return df_metrics

"""## Utility Functions"""

def create_directory(directory_path):
    if os.path.exists(directory_path):
        return None
    else:
        try:
            os.makedirs(directory_path)
        except:
            # in case another machine created the path meanwhile !:(
            return None
        return directory_path


def create_path(root_dir, classifier_name, archive_name):
    output_directory = root_dir + '/results/' + classifier_name + '/' + archive_name + '/'
    if os.path.exists(output_directory):
        return None
    else:
        os.makedirs(output_directory)
        return output_directory

def calculate_metrics(y_true, y_pred, duration, y_true_val=None, y_pred_val=None):
    res = pd.DataFrame(data=np.zeros((1, 4), dtype=np.float), index=[0],
                       columns=['precision', 'accuracy', 'recall', 'duration'])
    res['precision'] = precision_score(y_true, y_pred, average='macro')
    res['accuracy'] = accuracy_score(y_true, y_pred)

    if not y_true_val is None:
        # this is useful when transfer learning is used with cross validation
        res['accuracy_val'] = accuracy_score(y_true_val, y_pred_val)

    res['recall'] = recall_score(y_true, y_pred, average='macro')
    res['duration'] = duration
    return res


def save_test_duration(file_name, test_duration):
    res = pd.DataFrame(data=np.zeros((1, 1), dtype=np.float), index=[0],
                       columns=['test_duration'])
    res['test_duration'] = test_duration
    res.to_csv(file_name, index=False)


def generate_results_csv(output_file_name, root_dir):
    res = pd.DataFrame(data=np.zeros((0, 7), dtype=np.float), index=[],
                       columns=['classifier_name', 'archive_name', 'dataset_name',
                                'precision', 'accuracy', 'recall', 'duration'])
    for classifier_name in CLASSIFIERS:
        for archive_name in ARCHIVE_NAMES:
            datasets_dict = read_all_datasets(root_dir, archive_name)
            for it in range(ITERATIONS):
                curr_archive_name = archive_name
                if it != 0:
                    curr_archive_name = curr_archive_name + '_itr_' + str(it)
                for dataset_name in datasets_dict.keys():
                    output_dir = root_dir + '/results/' + classifier_name + '/' \
                                 + curr_archive_name + '/' + dataset_name + '/' + 'df_metrics.csv'
                    if not os.path.exists(output_dir):
                        continue
                    df_metrics = pd.read_csv(output_dir)
                    df_metrics['classifier_name'] = classifier_name
                    df_metrics['archive_name'] = archive_name
                    df_metrics['dataset_name'] = dataset_name
                    res = pd.concat((res, df_metrics), axis=0, sort=False)

    res.to_csv(root_dir + output_file_name, index=False)
    # aggreagte the accuracy for iterations on same dataset
    res = pd.DataFrame({
        'accuracy': res.groupby(
            ['classifier_name', 'archive_name', 'dataset_name'])['accuracy'].mean()
    }).reset_index()

    return res


def plot_epochs_metric(hist, file_name, metric='loss'):
    plt.figure()
    plt.plot(hist.history[metric])
    plt.plot(hist.history['val_' + metric])
    plt.title('model ' + metric)
    plt.ylabel(metric, fontsize='large')
    plt.xlabel('epoch', fontsize='large')
    plt.legend(['train', 'val'], loc='upper left')
    plt.savefig(file_name, bbox_inches='tight')
    plt.close()


def save_logs_t_leNet(output_directory, hist, y_pred, y_true, duration):
    hist_df = pd.DataFrame(hist.history)
    hist_df.to_csv(output_directory + 'history.csv', index=False)

    df_metrics = calculate_metrics(y_true, y_pred, duration)
    df_metrics.to_csv(output_directory + 'df_metrics.csv', index=False)

    index_best_model = hist_df['loss'].idxmin()
    row_best_model = hist_df.loc[index_best_model]

    df_best_model = pd.DataFrame(data=np.zeros((1, 6), dtype=np.float), index=[0],
                                 columns=['best_model_train_loss', 'best_model_val_loss', 'best_model_train_acc',
                                          'best_model_val_acc', 'best_model_learning_rate', 'best_model_nb_epoch'])

    df_best_model['best_model_train_loss'] = row_best_model['loss']
    df_best_model['best_model_val_loss'] = row_best_model['val_loss']
    df_best_model['best_model_train_acc'] = row_best_model['acc']
    df_best_model['best_model_val_acc'] = row_best_model['val_acc']
    df_best_model['best_model_nb_epoch'] = index_best_model

    df_best_model.to_csv(output_directory + 'df_best_model.csv', index=False)

    # plot losses
    plot_epochs_metric(hist, output_directory + 'epochs_loss.png')


def save_logs(output_directory, hist, y_pred, y_true, duration, lr=True, y_true_val=None, y_pred_val=None):
    hist_df = pd.DataFrame(hist.history)
    hist_df.to_csv(output_directory + 'history.csv', index=False)

    df_metrics = calculate_metrics(y_true, y_pred, duration, y_true_val, y_pred_val)
    df_metrics.to_csv(output_directory + 'df_metrics.csv', index=False)

    index_best_model = hist_df['loss'].idxmin()
    row_best_model = hist_df.loc[index_best_model]

    df_best_model = pd.DataFrame(data=np.zeros((1, 6), dtype=np.float), index=[0],
                                 columns=['best_model_train_loss', 'best_model_val_loss', 'best_model_train_acc',
                                          'best_model_val_acc', 'best_model_learning_rate', 'best_model_nb_epoch'])

    df_best_model['best_model_train_loss'] = row_best_model['loss']
    df_best_model['best_model_val_loss'] = row_best_model['val_loss']
    df_best_model['best_model_train_acc'] = row_best_model['accuracy']
    df_best_model['best_model_val_acc'] = row_best_model['val_accuracy']
    df_best_model['best_model_nb_epoch'] = index_best_model

    df_best_model.to_csv(output_directory + 'df_best_model.csv', index=False)

    # for FCN there is no hyperparameters fine tuning - everything is static in code

    # plot losses
    plot_epochs_metric(hist, output_directory + 'epochs_loss.png')

    return df_metrics


def visualize_filter(root_dir):
    import tensorflow.keras as keras
    classifier = 'resnet'
    archive_name = 'UCRArchive_2018'
    dataset_name = 'GunPoint'
    datasets_dict = read_dataset(root_dir, archive_name, dataset_name)

    x_train = datasets_dict[dataset_name][0]
    y_train = datasets_dict[dataset_name][1]

    x_train = x_train.reshape(x_train.shape[0], x_train.shape[1], 1)

    model = keras.models.load_model(
        root_dir + 'results/' + classifier + '/' + archive_name + '/' + dataset_name + '/best_model.hdf5')

    # filters
    filters = model.layers[1].get_weights()[0]

    new_input_layer = model.inputs
    new_output_layer = [model.layers[1].output]

    new_feed_forward = keras.backend.function(new_input_layer, new_output_layer)

    classes = np.unique(y_train)

    colors = [(255 / 255, 160 / 255, 14 / 255), (181 / 255, 87 / 255, 181 / 255)]
    colors_conv = [(210 / 255, 0 / 255, 0 / 255), (27 / 255, 32 / 255, 101 / 255)]

    idx = 10
    idx_filter = 1

    filter = filters[:, 0, idx_filter]

    plt.figure(1)
    plt.plot(filter + 0.5, color='gray', label='filter')
    for c in classes:
        c_x_train = x_train[np.where(y_train == c)]
        convolved_filter_1 = new_feed_forward([c_x_train])[0]

        idx_c = int(c) - 1

        plt.plot(c_x_train[idx], color=colors[idx_c], label='class' + str(idx_c) + '-raw')
        plt.plot(convolved_filter_1[idx, :, idx_filter], color=colors_conv[idx_c], label='class' + str(idx_c) + '-conv')
        plt.legend()

    plt.savefig(root_dir + 'convolution-' + dataset_name + '.pdf')

    return 1


def viz_perf_themes(root_dir, df):
    df_themes = df.copy()
    themes_index = []
    # add the themes
    for dataset_name in df.index:
        themes_index.append(utils.constants.dataset_types[dataset_name])

    themes_index = np.array(themes_index)
    themes, themes_counts = np.unique(themes_index, return_counts=True)
    df_themes.index = themes_index
    df_themes = df_themes.rank(axis=1, method='min', ascending=False)
    df_themes = df_themes.where(df_themes.values == 1)
    df_themes = df_themes.groupby(level=0).sum(axis=1)
    df_themes['#'] = themes_counts

    for classifier in CLASSIFIERS:
        df_themes[classifier] = df_themes[classifier] / df_themes['#'] * 100
    df_themes = df_themes.round(decimals=1)
    df_themes.to_csv(root_dir + 'tab-perf-theme.csv')


def viz_perf_train_size(root_dir, df):
    df_size = df.copy()
    train_sizes = []
    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')
    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')
    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)

    for dataset_name in df.index:
        train_size = len(datasets_dict[dataset_name][0])
        train_sizes.append(train_size)

    train_sizes = np.array(train_sizes)
    bins = np.array([0, 100, 400, 800, 99999])
    train_size_index = np.digitize(train_sizes, bins)
    train_size_index = bins[train_size_index]

    df_size.index = train_size_index
    df_size = df_size.rank(axis=1, method='min', ascending=False)
    df_size = df_size.groupby(level=0, axis=0).mean()
    df_size = df_size.round(decimals=2)

    print(df_size.to_string())
    df_size.to_csv(root_dir + 'tab-perf-train-size.csv')


def viz_perf_classes(root_dir, df):
    df_classes = df.copy()
    class_numbers = []
    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')
    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')
    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)

    for dataset_name in df.index:
        train_size = len(np.unique(datasets_dict[dataset_name][1]))
        class_numbers.append(train_size)

    class_numbers = np.array(class_numbers)
    bins = np.array([0, 3, 4, 6, 8, 13, 9999])
    class_numbers_index = np.digitize(class_numbers, bins)
    class_numbers_index = bins[class_numbers_index]

    df_classes.index = class_numbers_index
    df_classes = df_classes.rank(axis=1, method='min', ascending=False)
    df_classes = df_classes.groupby(level=0, axis=0).mean()
    df_classes = df_classes.round(decimals=2)

    print(df_classes.to_string())
    df_classes.to_csv(root_dir + 'tab-perf-classes.csv')


def viz_perf_length(root_dir, df):
    df_lengths = df.copy()
    lengths = []
    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')
    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')
    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)

    for dataset_name in df.index:
        length = datasets_dict[dataset_name][0].shape[1]
        lengths.append(length)

    lengths = np.array(lengths)
    bins = np.array([0, 81, 251, 451, 700, 1001, 9999])
    lengths_index = np.digitize(lengths, bins)
    lengths_index = bins[lengths_index]

    df_lengths.index = lengths_index
    df_lengths = df_lengths.rank(axis=1, method='min', ascending=False)
    df_lengths = df_lengths.groupby(level=0, axis=0).mean()
    df_lengths = df_lengths.round(decimals=2)

    print(df_lengths.to_string())
    df_lengths.to_csv(root_dir + 'tab-perf-lengths.csv')


def viz_plot(root_dir, df):
    df_lengths = df.copy()
    lengths = []
    datasets_dict_ucr = read_all_datasets(root_dir, archive_name='UCR_TS_Archive_2015')
    datasets_dict_mts = read_all_datasets(root_dir, archive_name='mts_archive')
    datasets_dict = dict(datasets_dict_ucr, **datasets_dict_mts)

    for dataset_name in df.index:
        length = datasets_dict[dataset_name][0].shape[1]
        lengths.append(length)

    lengths_index = np.array(lengths)

    df_lengths.index = lengths_index

    plt.scatter(x=df_lengths['fcn'], y=df_lengths['resnet'])
    plt.ylim(ymin=0, ymax=1.05)
    plt.xlim(xmin=0, xmax=1.05)
    # df_lengths['fcn']
    plt.savefig(root_dir + 'plot.pdf')

"""## Calling Models"""

def fit_ecg1_classifier(classifier_name, output_directory):

    # Download the dataset
    dataframe = pd.read_csv('http://storage.googleapis.com/download.tensorflow.org/data/ecg.csv', header=None)
    raw_data = dataframe.values

    # The last element contains the labels
    labels = raw_data[:, -1]

    # The other data points are the electrocadriogram data
    data = raw_data[:, 0:-1]

    # Split the data further into training, validation, and test datasets
    test_size = 0.2
    randomseed = 12345

    x_train, x_test, y_train, y_test = train_test_split(data, labels, test_size=test_size, random_state=randomseed)

    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))

    # transform the labels from integers to one hot vectors
    enc = OneHotEncoder(categories='auto')
    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))
    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()
    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()

    # save orignal y because later we will use binary
    y_true = np.argmax(y_test, axis=1)

    if len(x_train.shape) == 2:  # if univariate
        # add a dimension to make it multivariate with one dimension 
        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))

    input_shape = x_train.shape[1:]
    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)

    classifier.fit(x_train, y_train, x_test, y_test, y_true)



def fit_ecg200_classifier(classifier_name, output_directory):

        # Download the dataset
    train = pd.read_csv('datasets/ecg200/ECG200_TRAIN.txt', delimiter='  ', header=None)

    numeric = train.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values

    train.head()
    train.describe()

    # The last element contains the labels
    y_train = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_train = raw_data[:, 1:]

    test = pd.read_csv('datasets/ecg200/ECG200_TEST.txt', delimiter='  ', header=None)

    numeric = test.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values
    #dataframe.head()
    test.describe()

    # The last element contains the labels
    y_test = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_test = raw_data[:, 1:]

    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))

    # transform the labels from integers to one hot vectors
    enc = OneHotEncoder(categories='auto')
    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))
    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()
    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()

    # save orignal y because later we will use binary
    y_true = np.argmax(y_test, axis=1)

    if len(x_train.shape) == 2:  # if univariate
        # add a dimension to make it multivariate with one dimension 
        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))

    input_shape = x_train.shape[1:]
    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)

    classifier.fit(x_train, y_train, x_test, y_test, y_true)


def fit_ecg5000_classifier(classifier_name, output_directory):

        # Download the dataset
    train = pd.read_csv('datasets/ecg5000/ECG5000_TRAIN.txt', delimiter='  ', header=None)

    numeric = train.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values

    train.head()
    train.describe()

    # The last element contains the labels
    y_train = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_train = raw_data[:, 1:]

    test = pd.read_csv('datasets/ecg5000/ECG5000_TEST.txt', delimiter='  ', header=None)

    numeric = test.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values
    #dataframe.head()
    test.describe()

    # The last element contains the labels
    y_test = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_test = raw_data[:, 1:]

    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))

    # transform the labels from integers to one hot vectors
    enc = OneHotEncoder(categories='auto')
    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))
    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()
    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()

    # save orignal y because later we will use binary
    y_true = np.argmax(y_test, axis=1)

    if len(x_train.shape) == 2:  # if univariate
        # add a dimension to make it multivariate with one dimension 
        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))

    input_shape = x_train.shape[1:]
    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)

    classifier.fit(x_train, y_train, x_test, y_test, y_true)


def fit_ecgfivedays_classifier(classifier_name, output_directory):

        # Download the dataset
    train = pd.read_csv('datasets/ecgfivedays/ECGFiveDays_TRAIN.txt', delimiter='  ', header=None)

    numeric = train.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values

    train.head()
    train.describe()

    # The last element contains the labels
    y_train = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_train = raw_data[:, 1:]

    test = pd.read_csv('datasets/ecgfivedays/ECGFiveDays_TEST.txt', delimiter='  ', header=None)

    numeric = test.apply(pd.to_numeric, args=('coerce',))


    raw_data = numeric.values
    #dataframe.head()
    test.describe()

    # The last element contains the labels
    y_test = raw_data[:, 0]

    # The other data points are the electrocadriogram data
    x_test = raw_data[:, 1:]

    nb_classes = len(np.unique(np.concatenate((y_train, y_test), axis=0)))

    # transform the labels from integers to one hot vectors
    enc = OneHotEncoder(categories='auto')
    enc.fit(np.concatenate((y_train, y_test), axis=0).reshape(-1, 1))
    y_train = enc.transform(y_train.reshape(-1, 1)).toarray()
    y_test = enc.transform(y_test.reshape(-1, 1)).toarray()

    # save orignal y because later we will use binary
    y_true = np.argmax(y_test, axis=1)

    if len(x_train.shape) == 2:  # if univariate
        # add a dimension to make it multivariate with one dimension 
        x_train = x_train.reshape((x_train.shape[0], x_train.shape[1], 1))
        x_test = x_test.reshape((x_test.shape[0], x_test.shape[1], 1))

    input_shape = x_train.shape[1:]
    classifier = create_classifier(classifier_name, input_shape, nb_classes, output_directory)

    classifier.fit(x_train, y_train, x_test, y_test, y_true)


def create_classifier(classifier_name, input_shape, nb_classes, output_directory, verbose=True):

    if classifier_name == 'mlp':
        return Classifier_MLP(output_directory, input_shape, nb_classes, verbose)
    if classifier_name == 'resnet':
        return Classifier_RESNET(output_directory, input_shape, nb_classes, verbose)
    if classifier_name == 'mcnn':
        return Classifier_MCNN(output_directory, verbose)
    if classifier_name == 'tlenet':
        return Classifier_TLENET(output_directory, verbose)
    if classifier_name == 'encoder':
        return Classifier_ENCODER(output_directory, input_shape, nb_classes, verbose)

# """### ECG 1 - MLP"""

# fit_ecg1_classifier('mlp', 'results/ecg1/mlp/')

# """### RESNET"""

# fit_ecg1_classifier('resnet', 'results/ecg1/resnet/')

# """### MCNN"""


# """### t-lenet"""

#fit_ecg1_classifier('encoder', 'results/ecg1/encoder/')

# But its actually 3, because 0
iterations = 2

models = ['mlp', 'resnet', 'mcnn', 'tlenet', 'encoder']



for i in range(iterations):

    # iterate over each model
    for model in models:

         fit_ecg1_classifier(model, 'results' + str(i) + '/ecg1/' + model + '/')

    for model in models:

        fit_ecg200_classifier(model, 'results' + str(i) + '/ecg200/' + model + '/')

    for model in models:

        fit_ecg5000_classifier(model, 'results' + str(i) + '/ecg5000/' + model + '/')

    for model in models:

        fit_ecgfivedays_classifier(model, 'results' + str(i) + '/ecgfivedays/' + model + '/')