Preprocessing-RealWorld.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw to subject-activity based acc/gyro files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some subjects don't have all the activities and some and data from different runs\n",
    "activities = ['climbingdown', 'climbingup', 'jumping', 'lying', 'running', 'sitting', 'standing', 'walking',]\n",
    "subject_activities = {\n",
    "    'proband1':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband2':['climbingdown', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband3':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband4':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband5':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband6':['climbingdown', 'climbingup', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband7':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband8':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband9':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband10':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband11':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband12':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband13':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband14':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband15':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "}\n",
    "sensor_locations = ['chest',\n",
    "                    'forearm',\n",
    "                    'head',\n",
    "                    'shin',\n",
    "                    'thigh',\n",
    "                    'upperarm',\n",
    "                    'waist',]\n",
    "sensors = ['acc', 'gyr']\n",
    "sensor_names = {'acc':'acc', 'gyr':'Gyroscope'}\n",
    "subjects = [sub for sub in os.listdir('./data/realworld2016_dataset/') if 'proband' in sub]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_reference_times(subject, activity):\n",
    "    global sensors, sensor_locations, sensor_names\n",
    "    times = []\n",
    "    reference_sensors = []\n",
    "    reference_sensor_locations = []\n",
    "    root = './data/realworld2016_dataset/'+subject+'/data/'\n",
    "    for sensor in sensors:\n",
    "        for sensor_location in sensor_locations:\n",
    "            path = root+sensor+'_'+activity+'_csv/'+sensor_names[sensor]+'_'+activity+'_'+sensor_location+'.csv'\n",
    "            df = pd.read_csv(path)\n",
    "            times.append(df['attr_time'].iloc[0])\n",
    "            reference_sensors.append(sensor)\n",
    "            reference_sensor_locations.append(sensor_location)\n",
    "    idx = np.where(np.max(times) == times)[0][0]\n",
    "    path = root+reference_sensors[idx]+'_'+activity+'_csv/'+sensor_names[reference_sensors[idx]]+'_'+activity+'_'+reference_sensor_locations[idx]+'.csv'\n",
    "    df = pd.read_csv(path)\n",
    "    reference_time = df['attr_time'].values[0]\n",
    "    for sensor in sensors:\n",
    "        for sensor_location in sensor_locations:\n",
    "            path = root+sensor+'_'+activity+'_csv/'+sensor_names[sensor]+'_'+activity+'_'+sensor_location+'.csv'\n",
    "            df = pd.read_csv(path)\n",
    "            times.append(df['attr_time'].iloc[-1])\n",
    "            reference_sensors.append(sensor)\n",
    "            reference_sensor_locations.append(sensor_location)\n",
    "    idx = np.where(np.min(times) == times)[0][0]\n",
    "    path = root+reference_sensors[idx]+'_'+activity+'_csv/'+sensor_names[reference_sensors[idx]]+'_'+activity+'_'+reference_sensor_locations[idx]+'.csv'\n",
    "    df = pd.read_csv(path)\n",
    "    apocalypse = df['attr_time'].values[-1]\n",
    "    return reference_time, apocalypse\n",
    "    \n",
    "class syncer:\n",
    "    def __init__(self, time_ref):\n",
    "        self.ts = time_ref\n",
    "        self.T = 20\n",
    "        \n",
    "    def find_sync_point(self, times):\n",
    "        diff = np.abs(times-self.ts)\n",
    "        if np.min(diff)>2560:\n",
    "            self.ts = self.ts+self.T\n",
    "            return np.nan\n",
    "        else:\n",
    "            index = np.where(diff==np.min(diff))[0][0]\n",
    "            self.ts = self.ts+self.T\n",
    "            return index\n",
    "    \n",
    "def generate_columns(sensors, sensor_locations):\n",
    "    columns = []\n",
    "    axes = ['TS', 'x', 'y', 'z']\n",
    "    for sensor_location in sensor_locations:\n",
    "        for sensor in sensors:\n",
    "            for axis in axes:\n",
    "                columns.append(sensor+'_'+sensor_location+'_'+axis)\n",
    "    return columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "cell_style": "center"
   },
   "outputs": [],
   "source": [
    "columns = generate_columns(sensors, sensor_locations)\n",
    "for subject in subjects:\n",
    "    for activity in subject_activities[subject]:\n",
    "        time_reference, apocalypse = find_reference_times(subject, activity)\n",
    "        length = int((apocalypse-time_reference)//20)\n",
    "        root = './data/realworld2016_dataset/'+subject+'/data/'\n",
    "        sensor_location_temp = []\n",
    "        for sensor_location in sensor_locations:\n",
    "            sensors_temp = []\n",
    "            for sensor in sensors:\n",
    "                sync = syncer(time_reference)\n",
    "                path = root+sensor+'_'+activity+'_csv/'+sensor_names[sensor]+'_'+activity+'_'+sensor_location+'.csv'\n",
    "                df = pd.read_csv(path)\n",
    "                sensor_temp = []\n",
    "                for idx in range(length):\n",
    "                    sync_point = sync.find_sync_point(df['attr_time'].values)\n",
    "                    if sync_point is np.nan:\n",
    "                        nan_array = np.empty(df.iloc[0, 1:].values.shape)\n",
    "                        nan_array[:] = np.nan\n",
    "                        sensor_temp.append(nan_array)\n",
    "                    else:\n",
    "                        sensor_temp.append(df.iloc[sync_point, 1:].values)\n",
    "                sensor_temp = np.stack(sensor_temp)\n",
    "                sensors_temp.append(sensor_temp)\n",
    "            sensors_temp = np.concatenate(sensors_temp, axis=1)\n",
    "            sensor_location_temp.append(sensors_temp)\n",
    "        sensor_location_temp = np.concatenate(sensor_location_temp, axis=1)\n",
    "        save_path = './data/realworld2016_dataset/'+subject+'/processed/'+subject+'_'+activity+'.csv'\n",
    "        storedf = pd.DataFrame(sensor_location_temp, columns = columns)\n",
    "        storedf.to_csv(save_path)\n",
    "        print('Subject: {}, Activity: {}, Saved to: {}'.format(subject, activity, save_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking splitted data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paths = ['G:/Projects/HAR/data/realworld2016_dataset/proband14/data/acc_climbingup_csv/acc_climbingup_waist.csv',\n",
    "#          'G:/Projects/HAR/data/realworld2016_dataset/proband14/data/acc_climbingup_csv/acc_climbingup_2_waist.csv',\n",
    "#          'G:/Projects/HAR/data/realworld2016_dataset/proband14/data/acc_climbingup_csv/acc_climbingup_3_waist.csv']\n",
    "# columns = pd.read_csv(paths[0]).columns[1:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = ['./data/realworld2016_dataset/proband7/data/gyr_climbingup_csv/Gyroscope_climbingup_waist.csv',\n",
    "         './data/realworld2016_dataset/proband7/data/gyr_climbingup_csv/Gyroscope_climbingup_2_waist.csv',\n",
    "         './data/realworld2016_dataset/proband7/data/gyr_climbingup_csv/Gyroscope_climbingup_3_waist.csv']\n",
    "columns = pd.read_csv(paths[0]).columns[1:].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.concatenate((pd.read_csv(paths[0]).values[:, 1:], pd.read_csv(paths[1]).values[:, 1:], pd.read_csv(paths[2]).values[:, 1:]), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(arr, columns=columns)\n",
    "df.to_csv(paths[0].split('/')[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NaN value check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allfiles = [os.path.join(path, name) for path, subdirs, files in os.walk('./data/realworld2016_dataset/') for name in files if (('processed' in path) and ('desktop.ini' not in name))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# True ../data/realworld2016_dataset/proband1\\processed\\proband1_climbingup.csv\n",
    "# True ../data/realworld2016_dataset/proband1\\processed\\proband1_standing.csv\n",
    "# True ../data/realworld2016_dataset/proband14\\processed\\proband14_climbingdown.csv\n",
    "# True ../data/realworld2016_dataset/proband14\\processed\\proband14_climbingup.csv\n",
    "# True ../data/realworld2016_dataset/proband15\\processed\\proband15_sitting.csv\n",
    "# True ../data/realworld2016_dataset/proband2\\processed\\proband2_running.csv\n",
    "# True ../data/realworld2016_dataset/proband2\\processed\\proband2_walking.csv\n",
    "# True ../data/realworld2016_dataset/proband4\\processed\\proband4_climbingdown.csv\n",
    "# True ../data/realworld2016_dataset/proband4\\processed\\proband4_climbingup.csv\n",
    "# True ../data/realworld2016_dataset/proband5\\processed\\proband5_climbingup.csv\n",
    "# True ../data/realworld2016_dataset/proband7\\processed\\proband7_climbingdown.csv\n",
    "# True ../data/realworld2016_dataset/proband7\\processed\\proband7_climbingup.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pd.read_csv('../data/realworld2016_dataset/proband5\\processed\\proband5_climbingup.csv').isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subject/activity to window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import h5py\n",
    "import tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activities = ['climbingdown', 'climbingup', 'jumping', 'lying', 'running', 'sitting', 'standing', 'walking',]\n",
    "subject_activities = {\n",
    "    'proband1':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband2':['climbingdown', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband3':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband4':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband5':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband6':['climbingdown', 'climbingup', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband7':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband8':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband9':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband10':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband11':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband12':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband13':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband14':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband15':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "}\n",
    "paths = ['./data/realworld2016_dataset/proband3/processed/',\n",
    "         './data/realworld2016_dataset/proband5/processed/',\n",
    "         './data/realworld2016_dataset/proband6/processed/',\n",
    "         './data/realworld2016_dataset/proband8/processed/',\n",
    "         './data/realworld2016_dataset/proband9/processed/',\n",
    "         './data/realworld2016_dataset/proband10/processed/',\n",
    "         './data/realworld2016_dataset/proband11/processed/',\n",
    "         './data/realworld2016_dataset/proband12/processed/',\n",
    "         './data/realworld2016_dataset/proband13/processed/',\n",
    "         './data/realworld2016_dataset/proband15/processed/',]\n",
    "labels = {'climbingdown':0, 'climbingup':1, 'jumping':2, 'lying':3, 'running':4, 'sitting':5, 'standing':6, 'walking':7}\n",
    "sensor_locations = ['chest',\n",
    "                    'forearm',\n",
    "                    'head',\n",
    "                    'shin',\n",
    "                    'thigh',\n",
    "                    'upperarm',\n",
    "                    'waist',]\n",
    "sensors = ['acc', 'gyr']\n",
    "sensor_names = {'acc':'acc', 'gyr':'Gyroscope'}\n",
    "subjects = ['proband10',\n",
    "            'proband11',\n",
    "            'proband12',\n",
    "            'proband13',\n",
    "            'proband15',\n",
    "            'proband3',\n",
    "            'proband5',\n",
    "            'proband6',\n",
    "            'proband8',\n",
    "            'proband9']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data_columns(sensors, sensor_locations):\n",
    "    columns = []\n",
    "    axes = ['x', 'y', 'z']\n",
    "    for sensor_location in sensor_locations:\n",
    "        for sensor in sensors:\n",
    "            for axis in axes:\n",
    "                columns.append(sensor+'_'+sensor_location+'_'+axis)\n",
    "    return columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = 250\n",
    "sr = 0.8 # about half of the data will be cut later and that will results in the same overlapping we used in PAMAP2\n",
    "data_columns = generate_data_columns(sensors, sensor_locations)\n",
    "s = int((1-sr)*w)\n",
    "for subject in subjects:\n",
    "    X = []\n",
    "    y = []\n",
    "    for activity in subject_activities[subject]:\n",
    "        path = './data/realworld2016_dataset/'+subject+'/processed/'+subject+'_'+activity+'.csv'\n",
    "        label = labels[activity]\n",
    "        dataArr = pd.read_csv(path)[data_columns].iloc[750:-250, :].reset_index() # Skipping data where the subject is waiting for a signal to start performing the activity\n",
    "        dataArr.interpolate(inplace=True, limit=100)\n",
    "        for idx in range(0, dataArr.shape[0]-w, s):\n",
    "            y.append(label)\n",
    "            X.append(dataArr.iloc[idx:idx + w, :].values[:, 1:])\n",
    "    X = np.stack(X)\n",
    "    y = np.array(y)\n",
    "    hdf_handle = tables.open_file('./data/realworld2016_dataset/'+subject+'/processed/'+subject+'-1000.hdf', 'w')\n",
    "    dX = hdf_handle.create_array(hdf_handle.root, 'X', X)\n",
    "    dy = hdf_handle.create_array(hdf_handle.root, 'y', y)\n",
    "    hdf_handle.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subjects with missing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import h5py\n",
    "import tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data_columns(sensors, sensor_locations):\n",
    "    columns = []\n",
    "    axes = ['x', 'y', 'z']\n",
    "    for sensor_location in sensor_locations:\n",
    "        for sensor in sensors:\n",
    "            for axis in axes:\n",
    "                columns.append(sensor+'_'+sensor_location+'_'+axis)\n",
    "    return columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "activities = ['climbingdown', 'climbingup', 'jumping', 'lying', 'running', 'sitting', 'standing', 'walking',]\n",
    "subject_activities = {\n",
    "    'proband1':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband2':['climbingdown', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband3':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband4':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband5':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband6':['climbingdown', 'climbingup', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband7':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband8':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband9':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband10':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband11':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband12':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband13':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband14':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "    'proband15':['climbingdown', 'climbingup', 'jumping', 'lying',\n",
    "                'running', 'sitting', 'standing', 'walking',],\n",
    "}\n",
    "labels = {'climbingdown':0, 'climbingup':1, 'jumping':2, 'lying':3, 'running':4, 'sitting':5, 'standing':6, 'walking':7}\n",
    "sensor_locations = ['chest',\n",
    "                    'forearm',\n",
    "                    'head',\n",
    "                    'shin',\n",
    "                    'thigh',\n",
    "                    'upperarm',\n",
    "                    'waist',]\n",
    "sensors = ['acc', 'gyr']\n",
    "sensor_names = {'acc':'acc', 'gyr':'Gyroscope'}\n",
    "subjects = ['proband14',\n",
    "            'proband1',\n",
    "            'proband2',\n",
    "            'proband4',\n",
    "            'proband7']\n",
    "# 14, 1, 2, 4, 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = 250\n",
    "sr = 0.8\n",
    "data_columns = generate_data_columns(sensors, sensor_locations)\n",
    "s = int((1-sr)*w)\n",
    "for subject in subjects:\n",
    "    X = []\n",
    "    y = []\n",
    "    for activity in subject_activities[subject]:\n",
    "        path = './data/realworld2016_dataset/'+subject+'/processed/'+subject+'_'+activity+'.csv'\n",
    "        label = labels[activity]\n",
    "        dataArr = pd.read_csv(path)[data_columns].iloc[1000:, :].reset_index()\n",
    "        for idx in range(0, dataArr.shape[0]-w, s):\n",
    "            if np.isnan(dataArr.iloc[idx:idx + w, :].values[:, 1:]).any():\n",
    "                continue\n",
    "            else:\n",
    "                y.append(label)\n",
    "                X.append(dataArr.iloc[idx:idx + w, :].values[:, 1:])\n",
    "    X = np.stack(X)\n",
    "    y = np.array(y)\n",
    "    hdf_handle = tables.open_file('./data/realworld2016_dataset/'+subject+'/processed/'+subject+'-1000.hdf', 'w')\n",
    "    dX = hdf_handle.create_array(hdf_handle.root, 'X', X)\n",
    "    dy = hdf_handle.create_array(hdf_handle.root, 'y', y)\n",
    "    hdf_handle.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (torch14)",
   "language": "python",
   "name": "torch14"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}