Augmentation_Data_generation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Augmenting data to train Audio-IMU fused model\n",
    "\n",
    "# We generate augmented data with\n",
    "- 50ms, 100ms, 150ms, 200ms, 500ms, and 1000ms of shifts.\n",
    "- The shifting is done in the IMU modality.\n",
    "- Training labels are are aligned with the Audio Modality.\n",
    "- The shift are saved in the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:523: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:524: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/nesl/anaconda3/envs/py36/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:532: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import scipy.io\n",
    "\n",
    "from keras import optimizers\n",
    "from keras.optimizers import SGD\n",
    "from keras.optimizers import Adam\n",
    "from keras.metrics import categorical_crossentropy\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.convolutional import *\n",
    "from keras.callbacks import Callback\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "\n",
    "\n",
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, LSTM, Dense, Dropout, Flatten, Activation\n",
    "from keras.layers.core import Permute, Reshape\n",
    "from keras import backend as K\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image as img_PIL\n",
    "\n",
    "\n",
    "# random seed.\n",
    "rand_seed = 2\n",
    "\n",
    "from numpy.random import seed\n",
    "seed(rand_seed)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(rand_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encoding(y_data):\n",
    "    Mapping=dict()\n",
    "    sub_dirs=['downstair','upstair','run','jump','walk','handwashing','exercise']\n",
    "\n",
    "    categories=7\n",
    "\n",
    "    count=0\n",
    "    for i in sub_dirs:\n",
    "        Mapping[i]=count\n",
    "        count=count+1\n",
    "\n",
    "    y_features2=[]\n",
    "    for i in range(len(y_data)):\n",
    "        Type=y_data[i]\n",
    "        lab=Mapping[Type]\n",
    "        y_features2.append(lab)\n",
    "\n",
    "    y_features=np.array(y_features2)\n",
    "    y_features=y_features.reshape(y_features.shape[0],1)\n",
    "    from keras.utils import to_categorical\n",
    "    y_features = to_categorical(y_features)\n",
    "\n",
    "    return y_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path='data/'\n",
    "\n",
    "\n",
    "def get_train_data(path=path):\n",
    "    Train_data=np.load(path+'train_5000.npz')\n",
    "    Features_imu=Train_data['arr_0']\n",
    "    Labels=Train_data['arr_1']\n",
    "    Features_sound=Train_data['arr_2']\n",
    "\n",
    "    Labels = one_hot_encoding(Labels)\n",
    "    Features_imu = Features_imu.reshape(Features_imu.shape[0],1, Features_imu.shape[1], Features_imu.shape[2]) \n",
    "\n",
    "    return Features_imu,Labels,Features_sound\n",
    "\n",
    "def get_valid_data(path=path):\n",
    "    Train_data=np.load(path+'valid_1000.npz')\n",
    "    Features_imu=Train_data['arr_0']\n",
    "    Labels=Train_data['arr_1']\n",
    "    Features_sound=Train_data['arr_2']\n",
    "\n",
    "    Labels = one_hot_encoding(Labels)\n",
    "    Features_imu = Features_imu.reshape(Features_imu.shape[0],1, Features_imu.shape[1], Features_imu.shape[2]) \n",
    "\n",
    "    return Features_imu,Labels,Features_sound\n",
    "\n",
    "def get_test_data(path=path):\n",
    "    Train_data=np.load(path+'test_1377.npz')\n",
    "    Features_imu=Train_data['arr_0']\n",
    "    Labels=Train_data['arr_1']\n",
    "    Features_sound=Train_data['arr_2']\n",
    "\n",
    "    Labels = one_hot_encoding(Labels)\n",
    "    Features_imu = Features_imu.reshape(Features_imu.shape[0],1, Features_imu.shape[1], Features_imu.shape[2]) \n",
    "\n",
    "    return Features_imu,Labels,Features_sound\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 1, 40, 12) (5000, 7) (5000, 193)\n",
      "Train Classes distribution:  [760. 994. 546. 780. 532. 732. 656.]\n",
      "(1000, 1, 40, 12) (1000, 7) (1000, 193)\n",
      "Valid Classes distribution:  [150. 188. 136. 141.  98. 157. 130.]\n",
      "(1377, 1, 40, 12) (1377, 7) (1377, 193)\n",
      "Test Classes distribution:  [219. 268. 124. 146. 143. 234. 243.]\n"
     ]
    }
   ],
   "source": [
    "Features_imu,Labels,Features_sound = get_train_data()\n",
    "print(Features_imu.shape, Labels.shape, Features_sound.shape)\n",
    "print('Train Classes distribution: ',np.sum(Labels, axis =0))\n",
    "\n",
    "Features_imu2,Labels2,Features_sound2 = get_valid_data()\n",
    "print(Features_imu2.shape, Labels2.shape, Features_sound2.shape)\n",
    "print('Valid Classes distribution: ',np.sum(Labels2, axis =0))\n",
    "\n",
    "Features_imu3,Labels3,Features_sound3 = get_test_data()\n",
    "print(Features_imu3.shape, Labels3.shape, Features_sound3.shape)\n",
    "print('Test Classes distribution: ',np.sum(Labels3, axis =0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observation: All of these test labels are continuous.\n",
    "### Time window used to create a sample: 2 Second."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Time shifting, our goal is to use the longest windows and insert time shifting into them"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# First preprocess the data to get the continuous acc and gyro windows\n",
    "- data ordering: 12 sensors each with 40 samples.\n",
    "- 12 sensors are: acc_right, gyro_right, acc_left, gyro_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_data2(path=path):\n",
    "    Train_data=np.load(path+'train_5000.npz')\n",
    "    Features_imu=Train_data['arr_0']\n",
    "    Labels=Train_data['arr_1']\n",
    "    Features_sound=Train_data['arr_2']\n",
    "    \n",
    "    #We will not do one hot encoding now\n",
    "    #Labels = one_hot_encoding(Labels)\n",
    "    \n",
    "    Features_imu = Features_imu.reshape(Features_imu.shape[0],1, Features_imu.shape[1], Features_imu.shape[2]) \n",
    "\n",
    "    return Features_imu,Labels,Features_sound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 1, 40, 12) (5000,) (5000, 193)\n"
     ]
    }
   ],
   "source": [
    "Features_imu_t1,Labels_t1,Features_sound_t1 = get_train_data2()\n",
    "print(Features_imu_t1.shape, Labels_t1.shape, Features_sound_t1.shape)\n",
    "#print('Test Classes distribution: ',np.sum(Labels_t1, axis =0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We see all of the labels are continuous\n",
    "Labels_t1 = np.array(Labels_t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['run', 'jump', 'handwashing', 'walk', 'handwashing', 'upstair',\n",
       "       'handwashing', 'downstair', 'handwashing', 'downstair', 'walk',\n",
       "       'upstair', 'run', 'run', 'walk', 'run', 'downstair', 'handwashing',\n",
       "       'walk', 'upstair', 'handwashing', 'exercise', 'exercise',\n",
       "       'handwashing', 'downstair', 'downstair', 'upstair', 'downstair',\n",
       "       'upstair', 'exercise', 'jump', 'downstair', 'upstair', 'upstair',\n",
       "       'exercise', 'upstair', 'handwashing', 'upstair', 'upstair',\n",
       "       'upstair', 'downstair', 'upstair', 'handwashing', 'exercise',\n",
       "       'handwashing', 'downstair', 'exercise', 'upstair', 'jump', 'run',\n",
       "       'walk', 'walk', 'upstair', 'handwashing', 'jump', 'exercise',\n",
       "       'jump', 'exercise', 'run', 'handwashing', 'walk', 'handwashing',\n",
       "       'handwashing', 'run', 'downstair', 'handwashing', 'run', 'upstair',\n",
       "       'handwashing', 'walk', 'handwashing', 'run', 'jump', 'handwashing',\n",
       "       'jump', 'run', 'downstair', 'jump', 'upstair', 'run',\n",
       "       'handwashing', 'walk', 'exercise', 'upstair', 'jump', 'run',\n",
       "       'walk', 'run', 'handwashing', 'upstair', 'upstair', 'upstair',\n",
       "       'upstair', 'jump', 'run', 'walk', 'handwashing', 'run',\n",
       "       'handwashing', 'run'], dtype='<U11')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Labels_t1[:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reorder the labels so they are continuous\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def number_labels(y_data):\n",
    "    Mapping=dict()\n",
    "    sub_dirs=['downstair','upstair','run','jump','walk','handwashing','exercise']\n",
    "\n",
    "    categories=10\n",
    "\n",
    "    count=0\n",
    "    for i in sub_dirs:\n",
    "        Mapping[i]=count\n",
    "        count=count+1\n",
    "\n",
    "    y_features2=[]\n",
    "    for i in range(len(y_data)):\n",
    "        Type=y_data[i]\n",
    "        lab=Mapping[Type]\n",
    "        y_features2.append(lab)\n",
    "\n",
    "    y_features=np.array(y_features2)\n",
    "    #y_features=y_features.reshape(y_features.shape[0],1)\n",
    "    return y_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "Labels_t1 = number_labels(Labels_t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2, 3, 5, 4, 5, 1, 5, 0, 5, 0])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Labels_t1[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 6, 6, 6])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a sorting order for the labels\n",
    "train_order = np.argsort(Labels_t1)\n",
    "Labels_t1[train_order]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use this sorting order to recreate the array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Train_imu=[]\n",
    "Train_labels=[]\n",
    "Train_sound=[]\n",
    "\n",
    "for i in range(Labels_t1.shape[0]):\n",
    "    Train_imu.append(Features_imu_t1[train_order[i]])\n",
    "    Train_labels.append(Labels_t1[train_order[i]])\n",
    "    Train_sound.append(Features_sound_t1[train_order[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 1, 40, 12) (5000,) (5000, 193)\n"
     ]
    }
   ],
   "source": [
    "Train_imu=np.array(Train_imu)\n",
    "Train_labels=np.array(Train_labels)\n",
    "Train_sound=np.array(Train_sound)\n",
    "\n",
    "print(Train_imu.shape, Train_labels.shape, Train_sound.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 6, 6, 6])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "Train_labels = to_categorical(Train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test accuracy of model on the newly concatenated data so we don't mess up the things\n",
    "\n",
    "from keras.models import load_model\n",
    "model_path = 'data/Baseline_IMU_Audio_Fusion'\n",
    "\n",
    "model = load_model(model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5000/5000 [==============================] - 5s 966us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.002210177281832057, 0.9994]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate([Train_sound,Train_imu],Train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing accuracy on Reshuffled data.\n",
    "## Note: not time errors are introduced so Accuracy should be same as the fusion model on original continuous test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now aggregating the data and doing the timing errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_right=[]\n",
    "gyro_right=[]\n",
    "acc_left=[]\n",
    "gyro_left=[]\n",
    "\n",
    "for i in range(Train_imu.shape[0]):\n",
    "    acc_right.append([Train_imu[i,0,:,0],Train_imu[i,0,:,1],Train_imu[i,0,:,2]])\n",
    "    gyro_right.append([Train_imu[i,0,:,3],Train_imu[i,0,:,4],Train_imu[i,0,:,5]])\n",
    "    acc_left.append([Train_imu[i,0,:,6],Train_imu[i,0,:,7],Train_imu[i,0,:,8]])\n",
    "    gyro_left.append([Train_imu[i,0,:,9],Train_imu[i,0,:,10],Train_imu[i,0,:,11]])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "acc_right=np.array(acc_right)\n",
    "gyro_right=np.array(gyro_right)\n",
    "acc_left=np.array(acc_left)\n",
    "gyro_left=np.array(gyro_left)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5000, 3, 40)\n",
      "(5000, 3, 40)\n",
      "(5000, 3, 40)\n",
      "(5000, 3, 40)\n"
     ]
    }
   ],
   "source": [
    "print(acc_right.shape)\n",
    "print(gyro_right.shape)\n",
    "print(acc_left.shape)\n",
    "print(gyro_left.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "acc_right_cont= acc_right[0,]\n",
    "gyro_right_cont= gyro_right[0,]\n",
    "\n",
    "acc_left_cont= acc_left[0,]\n",
    "gyro_left_cont= gyro_left[0,]\n",
    "\n",
    "\n",
    "for i in range(1, Features_imu_t1.shape[0]):\n",
    "    #print(i)\n",
    "    acc_right_cont=np.hstack((acc_right_cont,acc_right[i,]))\n",
    "    gyro_right_cont=np.hstack((gyro_right_cont,gyro_right[i,]))\n",
    "    acc_left_cont=np.hstack((acc_left_cont,acc_left[i,]))\n",
    "    gyro_left_cont=np.hstack((gyro_left_cont,gyro_left[i,]))\n",
    "\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 200000)\n",
      "(3, 200000)\n",
      "(3, 200000)\n",
      "(3, 200000)\n"
     ]
    }
   ],
   "source": [
    "print(acc_right_cont.shape)\n",
    "print(gyro_right_cont.shape)\n",
    "print(acc_left_cont.shape)\n",
    "print(gyro_left_cont.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Now shifting the samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Defining function to do the evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_sample_shift(shift_samples = 1):\n",
    "    sample_size = 40 #need to be 40, as decided during training\n",
    "    total_samples = acc_right_cont.shape[1]\n",
    "    \n",
    "    #print(total_samples)\n",
    "    \n",
    "    current_cursor = shift_samples\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    acc_right_pro= np.array(acc_right_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "\n",
    "    #print(acc_right_pro.shape)\n",
    "\n",
    "    gyro_right_pro= np.array(gyro_right_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "\n",
    "    acc_left_pro= np.array(acc_left_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "\n",
    "    gyro_left_pro= np.array(gyro_left_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "\n",
    "    #print(np.array(acc_right_cont[np.newaxis,:,current_cursor:current_cursor+sample_size]).shape)\n",
    "\n",
    "    while current_cursor<=(total_samples-2*sample_size):\n",
    "        current_cursor = current_cursor + sample_size\n",
    "        #print(current_cursor,\" : \", i)\n",
    "        a=acc_right_pro\n",
    "        b=np.array(acc_right_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "        acc_right_pro = np.concatenate((a,b),axis=0)\n",
    "\n",
    "\n",
    "        a=gyro_right_pro\n",
    "        b=np.array(gyro_right_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "        gyro_right_pro = np.concatenate((a,b),axis=0)\n",
    "\n",
    "\n",
    "        a=acc_left_pro\n",
    "        b=np.array(acc_left_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "        #print(a.shape,b.shape)\n",
    "        acc_left_pro = np.concatenate((a,b),axis=0)\n",
    "\n",
    "        a=gyro_left_pro\n",
    "        b=np.array(gyro_left_cont[np.newaxis,:,current_cursor:current_cursor+sample_size])\n",
    "        gyro_left_pro = np.concatenate((a,b),axis=0)\n",
    "        i = i+1\n",
    "        \n",
    "    IMU_processed = np.concatenate((acc_right_pro,gyro_right_pro,acc_left_pro,gyro_left_pro),axis=1)\n",
    "    IMU_processed = IMU_processed[:,np.newaxis,:,:]\n",
    "    IMU_processed = np.swapaxes(IMU_processed,2,3)\n",
    "        \n",
    "    size = IMU_processed.shape[0]    \n",
    "        \n",
    "    return Train_sound[:size], IMU_processed ,Train_labels[:size]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating more training datasets by doing shifting\n",
    "- 1 sample shift = 50 ms timing error.\n",
    "- 20 sample shift = 1000ms timing error.\n",
    "- 1000ms augmentation uses shifts (1,2,10,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4999, 193) (4999, 1, 40, 12) (4999, 7)\n"
     ]
    }
   ],
   "source": [
    "# #shift by 1 sample\n",
    "\n",
    "Features_sound_1, IMU_1 ,Labels_1 =  get_data_sample_shift(shift_samples = 1)\n",
    "print(Features_sound_1.shape, IMU_1.shape, Labels_1.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "path_1_sample='data/train_data_1_shift'\n",
    "\n",
    "np.savez(path_1_sample, IMU_1, Labels_1, Features_sound_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4999, 193) (4999, 1, 40, 12) (4999, 7)\n"
     ]
    }
   ],
   "source": [
    "#shift by 2 sample\n",
    "\n",
    "Features_sound_2, IMU_2 ,Labels_2 =  get_data_sample_shift(shift_samples = 2)\n",
    "print(Features_sound_2.shape, IMU_2.shape, Labels_2.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "path_2_sample='data/train_data_2_shift'\n",
    "\n",
    "np.savez(path_2_sample, IMU_2, Labels_2, Features_sound_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4999, 193) (4999, 1, 40, 12) (4999, 7)\n"
     ]
    }
   ],
   "source": [
    "#shift by 10 samples\n",
    "\n",
    "Features_sound_10, IMU_10 ,Labels_10 =  get_data_sample_shift(shift_samples = 10)\n",
    "print(Features_sound_10.shape, IMU_10.shape, Labels_10.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "path1='data/train_data_10_shift'\n",
    "\n",
    "np.savez(path1, IMU_10, Labels_10, Features_sound_10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4999, 193) (4999, 1, 40, 12) (4999, 7)\n"
     ]
    }
   ],
   "source": [
    "#shift by 20 samples\n",
    "\n",
    "Features_sound_20, IMU_20 ,Labels_20 =  get_data_sample_shift(shift_samples = 20)\n",
    "print(Features_sound_20.shape, IMU_20.shape, Labels_20.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "path2='data/train_data_20_shift'\n",
    "\n",
    "np.savez(path2, IMU_20, Labels_20, Features_sound_20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}