Data_Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.io as sio\n",
    "import scipy.interpolate\n",
    "import matplotlib.pyplot as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Update dataset name as needed\n",
    "\n",
    "RAW_DATA_FOLDER = 'raw_data/'\n",
    "RAW_MAT_FOLDER = RAW_DATA_FOLDER + 'raw_mat/'\n",
    "RAW_NPY_FOLDER = RAW_DATA_FOLDER + 'raw_npy/'\n",
    "NPY_FOLDER = 'processed_input_data/'\n",
    "\n",
    "#split data\n",
    "## attribute all exp_1 to training\n",
    "## attribute 2 windows (split_ratio) from exp_2 and exp_3 to training\n",
    "## the rest were randomly used for either eval and train. \n",
    "ML_exp = 'split_master'\n",
    "split_ratio=2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#EXP_LIST contains a list of raw experiment XXX.mat files \n",
    "#Format: raw .mat --> raw .npy --> sliced & labeled .npy \n",
    "\n",
    "EXP_LIST = ['EB_025_1','EB_025_2','EB_025_3',\n",
    "           'EB_050_1','EB_050_2','EB_050_3',\n",
    "           'EB_150_1','EB_150_2','EB_150_3',\n",
    "           'PP_025_1','PP_025_2',\n",
    "           'PP_050_1','PP_050_2',\n",
    "           'PP_100_1','PP_100_2',\n",
    "           'PP_150_1']\n",
    "\n",
    "\n",
    "\n",
    "#SLICE_PARAM = [initial_index, strike, win_H, win_W]\n",
    "SLICE_PARAM = [0, 50, 128, 64]\n",
    "\n",
    "#Constant for step size from image processing\n",
    "DX = 1.14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic functions to generate.npy file \n",
    "\n",
    "## dimension have to be adjusted if storing more values of the same dimension(x,y,t)\n",
    "def compile_cfuv(c,f,u,v):\n",
    "    ## composing a matrix of dimension (cfuv, x, y, t) \n",
    "    data = np.zeros((4, *c.shape))              \n",
    "    data[0] = c\n",
    "    data[1] = f\n",
    "    data[2] = u\n",
    "    data[3] = v\n",
    "    ##Transpose axes to (x, y, cfuv, t)\n",
    "    data = np.transpose(data, axes=(1, 2, 0, 3))\n",
    "    print(data.shape)\n",
    "    return data\n",
    "\n",
    "\n",
    "def create_npy(exp):\n",
    "    matlab_data_path = RAW_MAT_FOLDER + str(exp) + '.mat' \n",
    "    npy_data_path = RAW_NPY_FOLDER + str(exp) +'.npy'\n",
    "    \n",
    "    if not os.path.exists(RAW_NPY_FOLDER):\n",
    "        os.mkdir(RAW_NPY_FOLDER)\n",
    "    if os.path.exists(npy_data_path):\n",
    "        raise Exception(\"raw .npy file already exist\")\n",
    "    else:  \n",
    "        # storing value of each channel from file path and key (column head named from matlab)\n",
    "        curl_map = sio.loadmat(matlab_data_path)[\"curl_map\"]\n",
    "        store_u = sio.loadmat(matlab_data_path)[\"store_u\"]\n",
    "        store_v = sio.loadmat(matlab_data_path)[\"store_v\"]\n",
    "        altBW_filt = sio.loadmat(matlab_data_path) [\"altBW_filt\"]    \n",
    "        # compile and structure to (x,y,cfuv_channel,t) and save\n",
    "        data = compile_cfuv(curl_map, altBW_filt, store_u, store_v)\n",
    "        np.save(npy_data_path, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate and save raw_npy for list of experiment\n",
    "# Only need to run once! \n",
    "for e in EXP_LIST:\n",
    "    print(e)\n",
    "    create_npy(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function that slice npy file into appropriate size of win_H and win_W\n",
    "## SLICE_PARAM = [ini_H=0, strike=50, win_H=128, win_W=64]\n",
    "## Raw npy is of size H=135, W=313 with 4 channel (cfuv) and stacking times\n",
    "## 'strike' allows some overlap of sliced npy file \n",
    "\n",
    "\n",
    "def slicing_hw(RAW_NPY_FOLDER, exp, slice_param): \n",
    "    npy_data_path = RAW_NPY_FOLDER + str(exp) +'.npy'\n",
    "    data = np.load(npy_data_path)\n",
    "    \n",
    "    #data has dimension of (x, y, cfuv, t)\n",
    "    width_tot = data.shape[1]\n",
    "    t_tot = data.shape[3]\n",
    "    \n",
    "    #unpack slice_param into initial_index, strike_lenght, window_H and window_W\n",
    "    ii = slice_param[0]\n",
    "    strike = slice_param[1]\n",
    "    H = slice_param[2]\n",
    "    W = slice_param[3]\n",
    "    \n",
    "    # Current experiments yield ~5 windows with a fixed slice_param \n",
    "    num_win = 1 + (width_tot - W)//strike   \n",
    "    data_h = data[ii: ii+H]\n",
    "    data_hw = []\n",
    "    w_idx = np.arange(num_win)\n",
    "    for w_i in w_idx:\n",
    "        left_win = w_i*strike\n",
    "        data_hwi = data_h[:,left_win:left_win+W]\n",
    "        \n",
    "        #data_hw is a list of tuples (number of tuples = num_win)\n",
    "        #first element of tuple has a shape = (128, 64, 4, 228)\n",
    "        #second element of tuple is the w_idx\n",
    "        data_hw.append((data_hwi, w_i))\n",
    "    \n",
    "    print(\"slice_shape =\", data_hw[0][0].shape)\n",
    "    return data_hw, num_win, t_tot\n",
    "\n",
    "# define total velocity for each frame\n",
    "def vel_tot_cal(RAW_NPY_FOLDER, exp):\n",
    "    #calculate the displacement at the bottom and top of frame. The\n",
    "    #difference is the total velocity\n",
    "    npy_data_path = RAW_NPY_FOLDER + str(exp) +'.npy'\n",
    "    data = np.load(npy_data_path)\n",
    "    #each experiment have distinct tot_t or # of frame \n",
    "    t_tot = data.shape[3]\n",
    "    vel_tot =[]\n",
    "    for t in range(t_tot):\n",
    "        u = data[:,:,2,t]\n",
    "        v = data[:,:,3,t]\n",
    "        vel_ins = np.sqrt((np.median(u[0], axis=0)-np.median(u[-1], axis=0))**2\n",
    "                          +(np.median(v[0], axis=0)-np.median(v[-1], axis=0))**2)\n",
    "        #roughly in the unit of 0.25mm/30s\n",
    "        vel_tot.append(vel_ins)\n",
    "    #calculating one velocity per time frame \n",
    "    return vel_tot    \n",
    "\n",
    "# function that provide KE label for each slice of size 64 (width) x 128 (height)\n",
    "# KE is calcuated by total_CurlSlip/vel\n",
    "\n",
    "def data_slice_KE(RAW_NPY_FOLDER, NPY_FOLDER, exp, slice_param,step):\n",
    "    \n",
    "    #set file location for \n",
    "    npy_exp_folder_path = NPY_FOLDER + 'slice_npy/' + str(exp)\n",
    "    master_folder_path = NPY_FOLDER + 'file_master/'\n",
    "    \n",
    "\n",
    "    if os.path.exists(npy_exp_folder_path):\n",
    "        raise Exception(\"slice_exp.npy already exist\")\n",
    "    else: \n",
    "        os.makedirs(npy_exp_folder_path)\n",
    "        if not os.path.exists(master_folder_path): \n",
    "            os.makedirs(master_folder_path)     \n",
    "    \n",
    "    \n",
    "    data_hw, num_win, t_tot = slicing_hw(RAW_NPY_FOLDER, exp, slice_param)\n",
    "    vel_tot = vel_tot_cal(RAW_NPY_FOLDER, exp)\n",
    "    \n",
    "    w_idx = np.arange(num_win)\n",
    "    file_master = []\n",
    "    \n",
    "    #looping through all windows and times \n",
    "    for w_i in w_idx: \n",
    "        for t_i in range(t_tot):\n",
    "            #unpacking data_hw\n",
    "            data_s, win = data_hw[w_i]\n",
    "            SLICE = data_s[:, :, :, t_i]\n",
    "            c = data_s[:,:,0,t_i]\n",
    "            f = data_s[:,:,1,t_i]\n",
    "            \n",
    "            #only process slice with non-zero fault traces\n",
    "            if np.sum(f) != 0:\n",
    "                curl_slip = 2*np.trapz(c*f,dx=step,axis=0)\n",
    "                label_KE = format((np.median(curl_slip)/np.array(vel_tot)[t_i]), '.2f')\n",
    "                label_SD = format((np.std(curl_slip)/np.array(vel_tot)[t_i]), '.2f')\n",
    "                \n",
    "                name_path = '/' + str(label_KE) + '_' + str(label_SD) + '_win_' + str(\n",
    "                        win) + '_t_' + str('{:03d}'.format(t_i)) + '_' + str(exp) + '_cfuv.npy'\n",
    "                # save sliced and labeled dataset in slice_npy folder\n",
    "                np.save(npy_exp_folder_path + name_path, SLICE)\n",
    "                \n",
    "                # add to file master\n",
    "                file_master.append(str(exp) + name_path)          \n",
    "    \n",
    "    master_data_path = master_folder_path + str(exp) + \".txt\"\n",
    "    print('non_zero_slice =', len(file_master))\n",
    "    np.savetxt(master_data_path, file_master, fmt=\"%s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only need to run once! \n",
    "slice_param = SLICE_PARAM\n",
    "step = DX\n",
    "for exp in EXP_LIST:\n",
    "    data_slice_KE(RAW_NPY_FOLDER, NPY_FOLDER, exp, slice_param,step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dataset(DATA_FOLDER,ML_exp, G1, G2, split_ratio):\n",
    "    train_combine = []\n",
    "    eval_combine = []\n",
    "    test_combine = []\n",
    "    eval_exp_win_stat = []\n",
    "    test_exp_win_stat = []\n",
    "    train_exp_win_stat = []\n",
    "    for exp in G1:        \n",
    "        fmt = DATA_FOLDER + 'file_master/' + str(exp) + '.txt'\n",
    "        a = np.loadtxt(fmt, dtype=str)       \n",
    "        # for each experiments in G1, we 100% attribute them to training dataset. \n",
    "        train_combine = np.concatenate([train_combine, a])\n",
    "        # housekeeping for the exp/slice included in training\n",
    "        num_win = int(a[-1].split('win_')[-1][0:1])+1\n",
    "        for i in range (num_win):\n",
    "            train_slice = [exp, i]\n",
    "            train_exp_win_stat.append(train_slice)\n",
    "            \n",
    "    for exp in G2:\n",
    "        fmt = DATA_FOLDER + 'file_master/' + str(exp) + '.txt'\n",
    "        b = np.loadtxt(fmt, dtype=str)\n",
    "        #check last element for its window number + 1 to get #of window\n",
    "        num_win = int(b[-1].split('win_')[-1][0:1])+1\n",
    "        #check how many time slice per window \n",
    "        win_len = len(b)/num_win\n",
    "        \n",
    "        #check randomized which slice would be included in \n",
    "        shf_idx = np.arange(num_win)\n",
    "        print(shf_idx)\n",
    "        np.random.shuffle(shf_idx)\n",
    "        print(shf_idx)\n",
    "        \n",
    "        split_eval = split_ratio\n",
    "        \n",
    "        #distribute one random window-slice to eval\n",
    "        for idx in range(split_eval):\n",
    "            eval_ini = int(shf_idx[idx]*win_len)\n",
    "            eval_end = int((shf_idx[idx]+1)*win_len)\n",
    "            eval_combine = np.concatenate([eval_combine,  b[eval_ini:eval_end]])\n",
    "            # housekeeping for the exp/slice included in evaluating\n",
    "            eval_slice = [exp,shf_idx[idx]] \n",
    "            eval_exp_win_stat.append(eval_slice)\n",
    "                \n",
    "        #distribute one random window-slice to test\n",
    "        test_ini = int(shf_idx[split_eval]*win_len)\n",
    "        test_end = int((shf_idx[split_eval]+1)*win_len)\n",
    "        test_combine = np.concatenate([test_combine,  b[test_ini:test_end]])\n",
    "        # houseckeeping for the exp/slice included in testting\n",
    "        test_slice = [exp,shf_idx[split_eval]] \n",
    "        test_exp_win_stat.append(test_slice)\n",
    "        \n",
    "        #distribute the rest of random window-slices to train\n",
    "        for idx in range(split_eval+1,num_win):\n",
    "            train_ini = int(shf_idx[idx]*win_len)\n",
    "            train_end = int((shf_idx[idx]+1)*win_len)\n",
    "            train_combine = np.concatenate([train_combine, b[train_ini:train_end]])\n",
    "            # houseckeeping for the exp/slice included in training\n",
    "            train_slice = [exp,shf_idx[idx]] \n",
    "            train_exp_win_stat.append(train_slice)\n",
    "            \n",
    "    tot_dataset = len(train_combine) + len(eval_combine) + len(test_combine)\n",
    "    train_ratio = format(len(train_combine)/tot_dataset, '.2f')\n",
    "    eval_ratio = format(len(eval_combine)/tot_dataset, '.2f')\n",
    "    test_ratio = format(len(test_combine)/tot_dataset, '.2f')\n",
    "    \n",
    "    stat_dict = {'train_ew': train_exp_win_stat,\n",
    "                'eval_ew': eval_exp_win_stat,\n",
    "                'test_ew': test_exp_win_stat,\n",
    "                'tot_dataset':tot_dataset,\n",
    "                'train_ratio':train_ratio,\n",
    "                'eval_ratio':eval_ratio,\n",
    "                'test_ratio':test_ratio}\n",
    "    print(stat_dict)\n",
    "    ML_EXP_PATH = DATA_FOLDER + str(ML_exp)\n",
    "    if not os.path.exists(ML_EXP_PATH): \n",
    "        os.makedirs(ML_EXP_PATH)\n",
    "    \n",
    "    with open(ML_EXP_PATH +'/data_stat.txt', 'w') as f:\n",
    "        print(stat_dict, file=f)\n",
    "        \n",
    "    train_data_path = ML_EXP_PATH +  '/train_master.txt'\n",
    "    np.savetxt(train_data_path, train_combine, fmt=\"%s\")\n",
    "    \n",
    "    eval_data_path = ML_EXP_PATH +  '/eval_master.txt'\n",
    "    np.savetxt(eval_data_path, eval_combine, fmt=\"%s\")\n",
    "    \n",
    "    test_data_path = ML_EXP_PATH +  '/test_master.txt'\n",
    "    np.savetxt(test_data_path, test_combine, fmt=\"%s\")\n",
    "    return stat_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXP_LIST = ['EB_025_1','EB_025_2','EB_025_3',\n",
    "           'EB_050_1','EB_050_2','EB_050_3',\n",
    "           'EB_150_1','EB_150_2','EB_150_3',\n",
    "           'PP_025_1','PP_025_2',\n",
    "           'PP_050_1','PP_050_2',\n",
    "           'PP_100_1','PP_100_2',\n",
    "           'PP_150_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G1 = ['EB_025_1', 'EB_050_1', 'EB_150_1','PP_025_1','PP_050_1','PP_100_1']\n",
    "G2 = ['EB_025_2', 'EB_025_3', 'EB_050_2','EB_050_3','EB_150_2','EB_150_3','PP_025_2','PP_050_2','PP_100_2' ]\n",
    "create_dataset(NPY_FOLDER, ML_exp, G1, G2,split_ratio)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}