eyn-pre-cluster-unif.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['train_data.npy', 'trn_index_list.npy', 'custom.css', 'val_index_list.npy', '__results__.html', 'train_targets_inside.npy', 'submission.csv', 'train_targets.npy', 'test_ids.npy', '__output__.json', 'test_data.npy', '__notebook__.ipynb']\n"
     ]
    }
   ],
   "source": [
    "# place all the points on the map of last-seen-nonstat, excluding itself\n",
    "# folds information actually not necessary\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "print(os.listdir(\"../input/eyn-original\"))\n",
    "x_min, x_max = -1., 1.\n",
    "y_min, y_max = -.3, .3\n",
    "\n",
    "QUERY_SIZE = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 11) (134063, 2)\n",
      "(134063,) (36380, 1)\n",
      "(33515, 21, 11)\n"
     ]
    }
   ],
   "source": [
    "train_data = np.load(\"../input/eyn-original/train_data.npy\")\n",
    "test_data = np.load(\"../input/eyn-original/test_data.npy\")\n",
    "train_targets = np.load(\"../input/eyn-original/train_targets.npy\")\n",
    "train_targets_inside = np.load(\"../input/eyn-original/train_targets_inside.npy\")\n",
    "train_targets_inside_indexes = np.argwhere(train_targets_inside == 1)\n",
    "print(train_data.shape, train_targets.shape)\n",
    "print(train_targets_inside.shape, train_targets_inside_indexes.shape)\n",
    "print(test_data.shape)\n",
    "# 't_entry','t_exit','x_entry','y_entry','x_exit','y_exit','vmax','vmin','vmean','tid_0','tid_1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 4) (134063, 21, 4)\n",
      "(33515, 21, 4) (33515, 21, 4)\n"
     ]
    }
   ],
   "source": [
    "# the queries\n",
    "train_entry_loc = np.concatenate((train_data[:,:,2:4], \n",
    "                                  train_data[:,:,0:2]*0.0001 + \n",
    "                                  np.random.randn(*train_data[:,:,0:2].shape)*0.000001), axis=2)\n",
    "train_exit_loc = np.concatenate((train_data[:,:,4:6], train_data[:,:,0:2]*0.0001), axis=2)\n",
    "test_entry_loc = np.concatenate((test_data[:,:,2:4], test_data[:,:,0:2]*0.0001), axis=2)\n",
    "test_exit_loc = np.concatenate((test_data[:,:,4:6], test_data[:,:,0:2]*0.0001), axis=2)\n",
    "print(train_entry_loc.shape, train_exit_loc.shape)\n",
    "print(test_entry_loc.shape, test_exit_loc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(67065,)\n",
      "(67065, 4) (67065,)\n"
     ]
    }
   ],
   "source": [
    "# building the reference\n",
    "train_last_nonstat_indexes = np.load(\"../input/eyn-folds/train_last_not_stationary.npy\")\n",
    "print(train_last_nonstat_indexes.shape)\n",
    "train_last_nonstat_loc = train_entry_loc[train_last_nonstat_indexes, 0, :] \n",
    "train_last_nonstat_target = train_targets_inside[train_last_nonstat_indexes]\n",
    "print(train_last_nonstat_loc.shape, train_last_nonstat_target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import spatial\n",
    "\n",
    "reference = train_last_nonstat_loc\n",
    "reference_tree = spatial.KDTree(reference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 10) (134063, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "train_entry_query_dist, train_entry_query_indexes = reference_tree.query(train_entry_loc,k=QUERY_SIZE)\n",
    "print(train_entry_query_dist.shape, train_entry_query_indexes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 10) (134063, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "train_exit_query_dist, train_exit_query_indexes = reference_tree.query(train_exit_loc,k=QUERY_SIZE)\n",
    "print(train_exit_query_dist.shape, train_exit_query_indexes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33515, 21, 10) (33515, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "test_entry_query_dist, test_entry_query_indexes = reference_tree.query(test_entry_loc,k=QUERY_SIZE)\n",
    "print(test_entry_query_dist.shape, test_entry_query_indexes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(33515, 21, 10) (33515, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "test_exit_query_dist, test_exit_query_indexes = reference_tree.query(test_exit_loc,k=QUERY_SIZE)\n",
    "print(test_exit_query_dist.shape, test_exit_query_indexes.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 10) (134063, 21, 10)\n",
      "(33515, 21, 10) (33515, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "train_entry_query_target = np.take(train_last_nonstat_target, train_entry_query_indexes, mode = 'clip')\n",
    "train_exit_query_target = np.take(train_last_nonstat_target, train_exit_query_indexes, mode = 'clip')\n",
    "test_entry_query_target = np.take(train_last_nonstat_target, test_entry_query_indexes, mode = 'clip')\n",
    "test_exit_query_target = np.take(train_last_nonstat_target, test_exit_query_indexes, mode = 'clip')\n",
    "print(train_entry_query_target.shape, train_exit_query_target.shape)\n",
    "print(test_entry_query_target.shape, test_exit_query_target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final exit values should be nans\n",
    "train_exit_query_target[:,0,:] = np.nan\n",
    "train_exit_query_dist[:,0,:] = np.nan\n",
    "test_exit_query_target[:,0,:] = np.nan\n",
    "test_exit_query_dist[:,0,:] = np.nan\n",
    "\n",
    "# should exclude the last-seen-position\n",
    "train_entry_query_target[:,0,0] = np.nan\n",
    "train_entry_query_dist[:,0,0] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(134063, 21, 10) (134063, 21, 10)\n",
      "(134063, 21, 10) (134063, 21, 10)\n",
      "(33515, 21, 10) (33515, 21, 10)\n",
      "(33515, 21, 10) (33515, 21, 10)\n"
     ]
    }
   ],
   "source": [
    "print(train_entry_query_target.shape, train_exit_query_target.shape)\n",
    "print(train_entry_query_dist.shape, train_exit_query_dist.shape)\n",
    "print(test_entry_query_target.shape, test_exit_query_target.shape)\n",
    "print(test_entry_query_dist.shape, test_exit_query_dist.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_pickle(\"../input/eyn-pre-unravel-df/df_train.pickle\")\n",
    "df_test = pd.read_pickle(\"../input/eyn-pre-unravel-df/df_test.pickle\")\n",
    "df_original_columns = df_train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "entry_col_names_target = [\"at{}\".format(i) for i in range(QUERY_SIZE)]\n",
    "entry_col_names_dist = [\"ad{}\".format(i) for i in range(QUERY_SIZE)]\n",
    "exit_col_names_target = [\"bt{}\".format(i) for i in range(QUERY_SIZE)]\n",
    "exit_col_names_dist = [\"bd{}\".format(i) for i in range(QUERY_SIZE)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,(a,b,c,d) in enumerate(zip(entry_col_names_target, exit_col_names_target,\n",
    "                                 entry_col_names_dist, exit_col_names_dist)):\n",
    "    df_train[a] = train_entry_query_target[:,:,i].ravel()\n",
    "    df_train[c] = train_entry_query_dist[:,:,i].ravel()\n",
    "    df_train[b] = train_exit_query_target[:,:,i].ravel()\n",
    "    df_train[d] = train_exit_query_dist[:,:,i].ravel()\n",
    "    \n",
    "    df_test[a] = test_entry_query_target[:,:,i].ravel()\n",
    "    df_test[c] = test_entry_query_dist[:,:,i].ravel()\n",
    "    df_test[b] = test_exit_query_target[:,:,i].ravel()\n",
    "    df_test[d] = test_exit_query_dist[:,:,i].ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reinforce nan values for nan entries\n",
    "df_train.loc[np.isnan(train_data[:,:,2].ravel()), :] = np.nan\n",
    "df_test.loc[np.isnan(test_data[:,:,2].ravel()), :] = np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>t_entry</th>\n",
       "      <th>t_exit</th>\n",
       "      <th>x_entry</th>\n",
       "      <th>y_entry</th>\n",
       "      <th>x_exit</th>\n",
       "      <th>y_exit</th>\n",
       "      <th>vmax</th>\n",
       "      <th>vmin</th>\n",
       "      <th>vmean</th>\n",
       "      <th>tid_0</th>\n",
       "      <th>tid_1</th>\n",
       "      <th>entry_in</th>\n",
       "      <th>exit_in</th>\n",
       "      <th>dur</th>\n",
       "      <th>dur_a</th>\n",
       "      <th>dist</th>\n",
       "      <th>dist_a</th>\n",
       "      <th>speed</th>\n",
       "      <th>speed_a</th>\n",
       "      <th>dir_x</th>\n",
       "      <th>dir_y</th>\n",
       "      <th>dir_x_a</th>\n",
       "      <th>dir_y_a</th>\n",
       "      <th>at0</th>\n",
       "      <th>ad0</th>\n",
       "      <th>bt0</th>\n",
       "      <th>bd0</th>\n",
       "      <th>at1</th>\n",
       "      <th>ad1</th>\n",
       "      <th>bt1</th>\n",
       "      <th>bd1</th>\n",
       "      <th>at2</th>\n",
       "      <th>ad2</th>\n",
       "      <th>bt2</th>\n",
       "      <th>bd2</th>\n",
       "      <th>at3</th>\n",
       "      <th>ad3</th>\n",
       "      <th>bt3</th>\n",
       "      <th>bd3</th>\n",
       "      <th>at4</th>\n",
       "      <th>ad4</th>\n",
       "      <th>bt4</th>\n",
       "      <th>bd4</th>\n",
       "      <th>at5</th>\n",
       "      <th>ad5</th>\n",
       "      <th>bt5</th>\n",
       "      <th>bd5</th>\n",
       "      <th>at6</th>\n",
       "      <th>ad6</th>\n",
       "      <th>bt6</th>\n",
       "      <th>bd6</th>\n",
       "      <th>at7</th>\n",
       "      <th>ad7</th>\n",
       "      <th>bt7</th>\n",
       "      <th>bd7</th>\n",
       "      <th>at8</th>\n",
       "      <th>ad8</th>\n",
       "      <th>bt8</th>\n",
       "      <th>bd8</th>\n",
       "      <th>at9</th>\n",
       "      <th>ad9</th>\n",
       "      <th>bt9</th>\n",
       "      <th>bd9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seq</th>\n",
       "      <th>traj</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"8\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>0.004194</td>\n",
       "      <td>0.030917</td>\n",
       "      <td>-1.595688</td>\n",
       "      <td>-0.429282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.026722</td>\n",
       "      <td>0.040611</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.037680</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.927831</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.094538</td>\n",
       "      <td>0.995521</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001438</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001978</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002254</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002761</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003468</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004074</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004421</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004642</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.036417</td>\n",
       "      <td>-0.036417</td>\n",
       "      <td>-1.599250</td>\n",
       "      <td>-0.466794</td>\n",
       "      <td>-1.599250</td>\n",
       "      <td>-0.466794</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.601278</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.050029</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.083205</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.200257</td>\n",
       "      <td>0.979743</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001809</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001809</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002164</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.670278</td>\n",
       "      <td>-0.637694</td>\n",
       "      <td>-1.602144</td>\n",
       "      <td>-0.533842</td>\n",
       "      <td>-1.609269</td>\n",
       "      <td>-0.515810</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032583</td>\n",
       "      <td>0.024028</td>\n",
       "      <td>0.019388</td>\n",
       "      <td>0.008491</td>\n",
       "      <td>0.595032</td>\n",
       "      <td>0.353365</td>\n",
       "      <td>-0.367464</td>\n",
       "      <td>0.930038</td>\n",
       "      <td>0.760436</td>\n",
       "      <td>0.649413</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002898</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006580</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007184</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000243</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007247</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000268</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007609</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000334</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007850</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.710778</td>\n",
       "      <td>-0.694306</td>\n",
       "      <td>-1.603369</td>\n",
       "      <td>-0.546593</td>\n",
       "      <td>-1.608601</td>\n",
       "      <td>-0.539355</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016472</td>\n",
       "      <td>0.046389</td>\n",
       "      <td>0.008931</td>\n",
       "      <td>0.261206</td>\n",
       "      <td>0.542170</td>\n",
       "      <td>5.630798</td>\n",
       "      <td>-0.585843</td>\n",
       "      <td>0.810425</td>\n",
       "      <td>-0.040913</td>\n",
       "      <td>0.999163</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001014</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000649</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001377</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000758</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001447</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001615</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001141</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001560</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001855</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001967</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001708</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002183</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002303</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.765722</td>\n",
       "      <td>-0.757167</td>\n",
       "      <td>-1.696432</td>\n",
       "      <td>-0.835619</td>\n",
       "      <td>-1.592682</td>\n",
       "      <td>-0.807581</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008556</td>\n",
       "      <td>0.020056</td>\n",
       "      <td>0.107472</td>\n",
       "      <td>1.967866</td>\n",
       "      <td>12.561630</td>\n",
       "      <td>98.120718</td>\n",
       "      <td>0.965368</td>\n",
       "      <td>0.260892</td>\n",
       "      <td>-0.324647</td>\n",
       "      <td>-0.945835</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.022404</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027182</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029146</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030046</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030737</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001509</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002791</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.031490</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032276</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005444</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032572</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-0.792472</td>\n",
       "      <td>-0.785778</td>\n",
       "      <td>-0.988774</td>\n",
       "      <td>1.449250</td>\n",
       "      <td>-1.057569</td>\n",
       "      <td>1.025657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006694</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.429143</td>\n",
       "      <td>NaN</td>\n",
       "      <td>64.104337</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.160309</td>\n",
       "      <td>-0.987067</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004895</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004749</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005385</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006550</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006419</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006867</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006927</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008668</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016590</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016716</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010598</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016822</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010867</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017924</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011113</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.023875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           t_entry    t_exit   x_entry    ...          ad9  bt9       bd9\n",
       "seq traj                                  ...                            \n",
       "0   0     0.004194  0.030917 -1.595688    ...     0.004642  NaN       NaN\n",
       "    1    -0.036417 -0.036417 -1.599250    ...     0.002164  0.0  0.002164\n",
       "    2    -0.670278 -0.637694 -1.602144    ...     0.000336  0.0  0.008035\n",
       "    3    -0.710778 -0.694306 -1.603369    ...     0.002303  0.0  0.002017\n",
       "    4    -0.765722 -0.757167 -1.696432    ...     0.032572  0.0  0.005448\n",
       "    5    -0.792472 -0.785778 -0.988774    ...     0.011113  0.0  0.023875\n",
       "    6          NaN       NaN       NaN    ...          NaN  NaN       NaN\n",
       "    7          NaN       NaN       NaN    ...          NaN  NaN       NaN\n",
       "\n",
       "[8 rows x 63 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>t_entry</th>\n",
       "      <th>t_exit</th>\n",
       "      <th>x_entry</th>\n",
       "      <th>y_entry</th>\n",
       "      <th>x_exit</th>\n",
       "      <th>y_exit</th>\n",
       "      <th>vmax</th>\n",
       "      <th>vmin</th>\n",
       "      <th>vmean</th>\n",
       "      <th>tid_0</th>\n",
       "      <th>tid_1</th>\n",
       "      <th>entry_in</th>\n",
       "      <th>exit_in</th>\n",
       "      <th>dur</th>\n",
       "      <th>dur_a</th>\n",
       "      <th>dist</th>\n",
       "      <th>dist_a</th>\n",
       "      <th>speed</th>\n",
       "      <th>speed_a</th>\n",
       "      <th>dir_x</th>\n",
       "      <th>dir_y</th>\n",
       "      <th>dir_x_a</th>\n",
       "      <th>dir_y_a</th>\n",
       "      <th>at0</th>\n",
       "      <th>ad0</th>\n",
       "      <th>bt0</th>\n",
       "      <th>bd0</th>\n",
       "      <th>at1</th>\n",
       "      <th>ad1</th>\n",
       "      <th>bt1</th>\n",
       "      <th>bd1</th>\n",
       "      <th>at2</th>\n",
       "      <th>ad2</th>\n",
       "      <th>bt2</th>\n",
       "      <th>bd2</th>\n",
       "      <th>at3</th>\n",
       "      <th>ad3</th>\n",
       "      <th>bt3</th>\n",
       "      <th>bd3</th>\n",
       "      <th>at4</th>\n",
       "      <th>ad4</th>\n",
       "      <th>bt4</th>\n",
       "      <th>bd4</th>\n",
       "      <th>at5</th>\n",
       "      <th>ad5</th>\n",
       "      <th>bt5</th>\n",
       "      <th>bd5</th>\n",
       "      <th>at6</th>\n",
       "      <th>ad6</th>\n",
       "      <th>bt6</th>\n",
       "      <th>bd6</th>\n",
       "      <th>at7</th>\n",
       "      <th>ad7</th>\n",
       "      <th>bt7</th>\n",
       "      <th>bd7</th>\n",
       "      <th>at8</th>\n",
       "      <th>ad8</th>\n",
       "      <th>bt8</th>\n",
       "      <th>bd8</th>\n",
       "      <th>at9</th>\n",
       "      <th>ad9</th>\n",
       "      <th>bt9</th>\n",
       "      <th>bd9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seq</th>\n",
       "      <th>traj</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"6\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>0.005889</td>\n",
       "      <td>0.017556</td>\n",
       "      <td>1.221697</td>\n",
       "      <td>0.940063</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.011667</td>\n",
       "      <td>0.133861</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.002757</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.020592</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.444227</td>\n",
       "      <td>-0.895914</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006460</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006684</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008206</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008763</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014911</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017428</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018382</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018608</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018749</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.019193</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.157417</td>\n",
       "      <td>-0.127972</td>\n",
       "      <td>1.248303</td>\n",
       "      <td>1.254617</td>\n",
       "      <td>1.222922</td>\n",
       "      <td>0.942532</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029444</td>\n",
       "      <td>0.018944</td>\n",
       "      <td>0.313115</td>\n",
       "      <td>0.164695</td>\n",
       "      <td>10.634110</td>\n",
       "      <td>8.693565</td>\n",
       "      <td>-0.081059</td>\n",
       "      <td>-0.996709</td>\n",
       "      <td>0.988186</td>\n",
       "      <td>-0.153258</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004577</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004462</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004788</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005697</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006291</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007225</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011261</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008460</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.012929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008797</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015698</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009841</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009887</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016076</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011830</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016533</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011843</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.242583</td>\n",
       "      <td>-0.176361</td>\n",
       "      <td>0.285834</td>\n",
       "      <td>0.254906</td>\n",
       "      <td>1.085554</td>\n",
       "      <td>1.279858</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.066222</td>\n",
       "      <td>0.021389</td>\n",
       "      <td>1.300030</td>\n",
       "      <td>1.174368</td>\n",
       "      <td>19.631323</td>\n",
       "      <td>54.905527</td>\n",
       "      <td>0.615155</td>\n",
       "      <td>0.788407</td>\n",
       "      <td>-0.803733</td>\n",
       "      <td>-0.594991</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001893</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001620</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003955</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.004190</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007061</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.004703</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007530</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009792</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.006355</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.013626</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006708</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014199</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.007460</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.014901</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.015777</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.263972</td>\n",
       "      <td>-0.263972</td>\n",
       "      <td>1.229712</td>\n",
       "      <td>0.953644</td>\n",
       "      <td>1.229712</td>\n",
       "      <td>0.953644</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.052222</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017715</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.339218</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.496436</td>\n",
       "      <td>0.868073</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003539</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003539</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003782</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003782</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004392</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004392</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004566</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004566</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006051</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006051</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007894</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007894</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009124</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009124</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010030</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011442</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.327861</td>\n",
       "      <td>-0.316194</td>\n",
       "      <td>1.251197</td>\n",
       "      <td>1.406247</td>\n",
       "      <td>1.220918</td>\n",
       "      <td>0.938266</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011667</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.468959</td>\n",
       "      <td>NaN</td>\n",
       "      <td>40.196508</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.064566</td>\n",
       "      <td>-0.997913</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000450</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007982</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000501</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008195</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003892</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009996</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016512</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007829</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017260</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008112</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018897</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008286</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.018978</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009713</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020278</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.020502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           t_entry    t_exit   x_entry    ...          ad9  bt9       bd9\n",
       "seq traj                                  ...                            \n",
       "0   0     0.005889  0.017556  1.221697    ...     0.019193  NaN       NaN\n",
       "    1    -0.157417 -0.127972  1.248303    ...     0.011843  0.0  0.016829\n",
       "    2    -0.242583 -0.176361  0.285834    ...     0.008330  0.0  0.016480\n",
       "    3    -0.263972 -0.263972  1.229712    ...     0.011442  0.0  0.011442\n",
       "    4    -0.327861 -0.316194  1.251197    ...     0.011159  0.0  0.020502\n",
       "    5          NaN       NaN       NaN    ...          NaN  NaN       NaN\n",
       "\n",
       "[6 rows x 63 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test.head(6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[entry_col_names_target] = df_train[entry_col_names_target].astype('category')\n",
    "df_train[exit_col_names_target] = df_train[exit_col_names_target].astype('category')\n",
    "df_test[entry_col_names_target] = df_test[entry_col_names_target].astype('category')\n",
    "df_test[exit_col_names_target] = df_test[exit_col_names_target].astype('category')\n",
    "df_train = df_train.drop(columns=df_original_columns)\n",
    "df_test = df_test.drop(columns=df_original_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>at0</th>\n",
       "      <th>ad0</th>\n",
       "      <th>bt0</th>\n",
       "      <th>bd0</th>\n",
       "      <th>at1</th>\n",
       "      <th>ad1</th>\n",
       "      <th>bt1</th>\n",
       "      <th>bd1</th>\n",
       "      <th>at2</th>\n",
       "      <th>ad2</th>\n",
       "      <th>bt2</th>\n",
       "      <th>bd2</th>\n",
       "      <th>at3</th>\n",
       "      <th>ad3</th>\n",
       "      <th>bt3</th>\n",
       "      <th>bd3</th>\n",
       "      <th>at4</th>\n",
       "      <th>ad4</th>\n",
       "      <th>bt4</th>\n",
       "      <th>bd4</th>\n",
       "      <th>at5</th>\n",
       "      <th>ad5</th>\n",
       "      <th>bt5</th>\n",
       "      <th>bd5</th>\n",
       "      <th>at6</th>\n",
       "      <th>ad6</th>\n",
       "      <th>bt6</th>\n",
       "      <th>bd6</th>\n",
       "      <th>at7</th>\n",
       "      <th>ad7</th>\n",
       "      <th>bt7</th>\n",
       "      <th>bd7</th>\n",
       "      <th>at8</th>\n",
       "      <th>ad8</th>\n",
       "      <th>bt8</th>\n",
       "      <th>bd8</th>\n",
       "      <th>at9</th>\n",
       "      <th>ad9</th>\n",
       "      <th>bt9</th>\n",
       "      <th>bd9</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>seq</th>\n",
       "      <th>traj</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"10\" valign=\"top\">0</th>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001438</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001978</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002254</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002459</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002761</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003468</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004074</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004421</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004642</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000724</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001282</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001341</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001809</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001809</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001930</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002164</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002898</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006580</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006816</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007184</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000243</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007247</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000268</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007351</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007609</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000334</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.007850</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001014</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000649</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001377</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000758</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001447</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000960</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001615</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001141</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001560</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001855</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001563</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001967</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001708</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002183</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002303</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016330</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.022404</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.027182</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000426</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.029146</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000736</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030046</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030737</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001509</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.030909</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.002791</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.031490</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.003119</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032276</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005444</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.032572</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004895</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.004749</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.005385</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006550</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006419</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008360</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006867</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.006927</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.008668</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016590</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.009664</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016716</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010598</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.016822</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.010867</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.017924</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011113</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.023875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          at0       ad0  bt0       bd0    ...     at9       ad9  bt9       bd9\n",
       "seq traj                                  ...                                 \n",
       "0   0     NaN       NaN  NaN       NaN    ...     0.0  0.004642  NaN       NaN\n",
       "    1     0.0  0.000724  0.0  0.000724    ...     0.0  0.002164  0.0  0.002164\n",
       "    2     0.0  0.000145  0.0  0.002898    ...     0.0  0.000336  0.0  0.008035\n",
       "    3     0.0  0.001014  0.0  0.000649    ...     0.0  0.002303  0.0  0.002017\n",
       "    4     0.0  0.016330  0.0  0.000426    ...     0.0  0.032572  0.0  0.005448\n",
       "    5     0.0  0.001168  0.0  0.004895    ...     0.0  0.011113  0.0  0.023875\n",
       "    6     NaN       NaN  NaN       NaN    ...     NaN       NaN  NaN       NaN\n",
       "    7     NaN       NaN  NaN       NaN    ...     NaN       NaN  NaN       NaN\n",
       "    8     NaN       NaN  NaN       NaN    ...     NaN       NaN  NaN       NaN\n",
       "    9     NaN       NaN  NaN       NaN    ...     NaN       NaN  NaN       NaN\n",
       "\n",
       "[10 rows x 40 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train.to_pickle(\"df_train_cluster.pickle\")\n",
    "df_test.to_pickle(\"df_test_cluster.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2815323, 40) (703815, 40)\n"
     ]
    }
   ],
   "source": [
    "# to load\n",
    "df_train = pd.read_pickle(\"df_train_cluster.pickle\")\n",
    "df_test = pd.read_pickle(\"df_test_cluster.pickle\")\n",
    "print(df_train.shape, df_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "134063\n",
      "133877\n",
      "67065\n",
      "67065\n"
     ]
    }
   ],
   "source": [
    "str_list = []\n",
    "for i in range(len(train_entry_query_dist)):\n",
    "    str_list.append(str([train_entry_query_dist[i,0,:]]))\n",
    "print(len(str_list))\n",
    "print(len(set(str_list)))\n",
    "print(len(np.array(str_list)[train_last_nonstat_indexes]))\n",
    "print(len(set(np.array(str_list)[train_last_nonstat_indexes])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=int64)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = np.argwhere((train_entry_query_indexes[:,0,0][train_last_nonstat_indexes] - np.arange(len(train_last_nonstat_indexes)))!=0)[:,0]\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_entry_query_indexes[:,0,0][train_last_nonstat_indexes][a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_entry_query_indexes[:,0,1][train_last_nonstat_indexes][a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([], dtype=int64)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_entry_query_indexes[:,0,2][train_last_nonstat_indexes][a]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}