preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib as plt\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# directory where all the data is present\n",
    "base_dir = \"D:\\\\something-something-project\\\\data\\\\\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "D:\\something-something-project\\data\\\n"
     ]
    }
   ],
   "source": [
    "print (base_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loading only training and validation json files\n",
    "train_vids_id = pd.read_json(base_dir + 'something-something-v2-train.json')\n",
    "validation_vids_id = pd.read_json(base_dir + 'something-something-v2-validation.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>template</th>\n",
       "      <th>placeholders</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>78687</td>\n",
       "      <td>holding potato next to vicks vaporub bottle</td>\n",
       "      <td>Holding [something] next to [something]</td>\n",
       "      <td>[potato, vicks vaporub bottle]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>42326</td>\n",
       "      <td>spreading margarine onto bread</td>\n",
       "      <td>Spreading [something] onto [something]</td>\n",
       "      <td>[margarine, bread]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100904</td>\n",
       "      <td>putting pen on a surface</td>\n",
       "      <td>Putting [something] on a surface</td>\n",
       "      <td>[pen]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80715</td>\n",
       "      <td>lifting up one end of bottle, then letting it ...</td>\n",
       "      <td>Lifting up one end of [something], then lettin...</td>\n",
       "      <td>[bottle]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>34899</td>\n",
       "      <td>holding bulb</td>\n",
       "      <td>Holding [something]</td>\n",
       "      <td>[bulb]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>184568</td>\n",
       "      <td>pushing strap camera from right to left</td>\n",
       "      <td>Pushing [something] from right to left</td>\n",
       "      <td>[strap camera]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>112783</td>\n",
       "      <td>spilling mouthwash onto roll on</td>\n",
       "      <td>Spilling [something] onto [something]</td>\n",
       "      <td>[mouthwash, roll on]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>108895</td>\n",
       "      <td>pushing spiderman invitaion so that it slightl...</td>\n",
       "      <td>Pushing [something] so that it slightly moves</td>\n",
       "      <td>[spiderman invitaion]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>169270</td>\n",
       "      <td>moving headset across a surface until it falls...</td>\n",
       "      <td>Moving [something] across a surface until it f...</td>\n",
       "      <td>[headset]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>66533</td>\n",
       "      <td>folding paper towel</td>\n",
       "      <td>Folding [something]</td>\n",
       "      <td>[paper towel]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>67698</td>\n",
       "      <td>taking a slice of onion</td>\n",
       "      <td>Taking [one of many similar things on the table]</td>\n",
       "      <td>[a slice of onion]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>60195</td>\n",
       "      <td>pretending to turn hat upside down</td>\n",
       "      <td>Pretending to turn [something] upside down</td>\n",
       "      <td>[hat]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>187992</td>\n",
       "      <td>pretending to take bottle from window side</td>\n",
       "      <td>Pretending to take [something] from [somewhere]</td>\n",
       "      <td>[bottle, window side]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>102808</td>\n",
       "      <td>moving color pencils up</td>\n",
       "      <td>Moving [something] up</td>\n",
       "      <td>[color pencils]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>218043</td>\n",
       "      <td>twisting (wringing) shirt wet until water come...</td>\n",
       "      <td>Twisting (wringing) [something] wet until wate...</td>\n",
       "      <td>[shirt]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>82056</td>\n",
       "      <td>holding a pencil next to tissue box</td>\n",
       "      <td>Holding [something] next to [something]</td>\n",
       "      <td>[a pencil, tissue box]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>10158</td>\n",
       "      <td>twisting a cloth</td>\n",
       "      <td>Twisting [something]</td>\n",
       "      <td>[a cloth]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>113667</td>\n",
       "      <td>taking tissue out of tissue box</td>\n",
       "      <td>Taking [something] out of [something]</td>\n",
       "      <td>[tissue, tissue box]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>134982</td>\n",
       "      <td>dropping headset in front of tools bag</td>\n",
       "      <td>Dropping [something] in front of [something]</td>\n",
       "      <td>[headset, tools bag]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>37055</td>\n",
       "      <td>throwing sunscreen in the air and letting it fall</td>\n",
       "      <td>Throwing [something] in the air and letting it...</td>\n",
       "      <td>[sunscreen]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>199780</td>\n",
       "      <td>lifting up one end of card without letting it ...</td>\n",
       "      <td>Lifting up one end of [something] without lett...</td>\n",
       "      <td>[card]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>122775</td>\n",
       "      <td>moving notebook towards the camera</td>\n",
       "      <td>Moving [something] towards the camera</td>\n",
       "      <td>[notebook]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>151369</td>\n",
       "      <td>moving coconut wafer up</td>\n",
       "      <td>Moving [something] up</td>\n",
       "      <td>[coconut wafer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>170911</td>\n",
       "      <td>showing that tumbler is empty</td>\n",
       "      <td>Showing that [something] is empty</td>\n",
       "      <td>[tumbler]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>25020</td>\n",
       "      <td>dropping biscuit behind razor</td>\n",
       "      <td>Dropping [something] behind [something]</td>\n",
       "      <td>[biscuit, razor]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>116814</td>\n",
       "      <td>throwing a stone against tree</td>\n",
       "      <td>Throwing [something] against [something]</td>\n",
       "      <td>[a stone, tree]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>122986</td>\n",
       "      <td>holding pencil over box</td>\n",
       "      <td>Holding [something] over [something]</td>\n",
       "      <td>[pencil, box]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>216433</td>\n",
       "      <td>moving a pouch and a mobile phone closer to ea...</td>\n",
       "      <td>Moving [something] and [something] closer to e...</td>\n",
       "      <td>[a pouch, a mobile phone]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>47323</td>\n",
       "      <td>pushing iron from right to left</td>\n",
       "      <td>Pushing [something] from right to left</td>\n",
       "      <td>[iron]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>15409</td>\n",
       "      <td>pushing coin so it spins</td>\n",
       "      <td>Pushing [something] so it spins</td>\n",
       "      <td>[coin]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>123183</td>\n",
       "      <td>covering a water bottle with a cloth</td>\n",
       "      <td>Covering [something] with [something]</td>\n",
       "      <td>[a water bottle, a cloth]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>208007</td>\n",
       "      <td>putting match into empty pill bottle</td>\n",
       "      <td>Putting [something] into [something]</td>\n",
       "      <td>[match, empty pill bottle]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>63908</td>\n",
       "      <td>showing that pen is inside bucket</td>\n",
       "      <td>Showing that [something] is inside [something]</td>\n",
       "      <td>[pen, bucket]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>156381</td>\n",
       "      <td>throwing headset</td>\n",
       "      <td>Throwing [something]</td>\n",
       "      <td>[headset]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>116989</td>\n",
       "      <td>pulling knife from right to left</td>\n",
       "      <td>Pulling [something] from right to left</td>\n",
       "      <td>[knife]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>114072</td>\n",
       "      <td>tipping disposable lighter over</td>\n",
       "      <td>Tipping [something] over</td>\n",
       "      <td>[disposable lighter]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>203597</td>\n",
       "      <td>moving tomato across a surface until it falls ...</td>\n",
       "      <td>Moving [something] across a surface until it f...</td>\n",
       "      <td>[tomato]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>14991</td>\n",
       "      <td>covering stone with paper</td>\n",
       "      <td>Covering [something] with [something]</td>\n",
       "      <td>[stone, paper]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>101299</td>\n",
       "      <td>pulling chopsticks from right to left</td>\n",
       "      <td>Pulling [something] from right to left</td>\n",
       "      <td>[chopsticks]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>186594</td>\n",
       "      <td>poking a stack of tiles without the stack coll...</td>\n",
       "      <td>Poking a stack of [something] without the stac...</td>\n",
       "      <td>[tiles]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>111183</td>\n",
       "      <td>letting marble roll along a flat surface</td>\n",
       "      <td>Letting [something] roll along a flat surface</td>\n",
       "      <td>[marble]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>105168</td>\n",
       "      <td>showing 'enak' vanilla canned milk behind 'tig...</td>\n",
       "      <td>Showing [something] behind [something]</td>\n",
       "      <td>['enak' vanilla canned milk, 'tiga sapi' choco...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>190776</td>\n",
       "      <td>dropping compass tool onto box</td>\n",
       "      <td>Dropping [something] onto [something]</td>\n",
       "      <td>[compass tool, box]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>181638</td>\n",
       "      <td>throwing a calculator in the air and catching it</td>\n",
       "      <td>Throwing [something] in the air and catching it</td>\n",
       "      <td>[a calculator]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>140721</td>\n",
       "      <td>throwing a ball against a sofa</td>\n",
       "      <td>Throwing [something] against [something]</td>\n",
       "      <td>[a ball, a sofa]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>145959</td>\n",
       "      <td>squeezing paper</td>\n",
       "      <td>Squeezing [something]</td>\n",
       "      <td>[paper]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>201957</td>\n",
       "      <td>pushing pencil off of book</td>\n",
       "      <td>Pushing [something] off of [something]</td>\n",
       "      <td>[pencil, book]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>20060</td>\n",
       "      <td>throwing card in the air and catching it</td>\n",
       "      <td>Throwing [something] in the air and catching it</td>\n",
       "      <td>[card]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>203403</td>\n",
       "      <td>moving polish and a box closer to each other</td>\n",
       "      <td>Moving [something] and [something] closer to e...</td>\n",
       "      <td>[polish, a box]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>133878</td>\n",
       "      <td>pushing newspaper so that it slightly moves</td>\n",
       "      <td>Pushing [something] so that it slightly moves</td>\n",
       "      <td>[newspaper]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                              label  \\\n",
       "0    78687        holding potato next to vicks vaporub bottle   \n",
       "1    42326                     spreading margarine onto bread   \n",
       "2   100904                           putting pen on a surface   \n",
       "3    80715  lifting up one end of bottle, then letting it ...   \n",
       "4    34899                                       holding bulb   \n",
       "5   184568            pushing strap camera from right to left   \n",
       "6   112783                    spilling mouthwash onto roll on   \n",
       "7   108895  pushing spiderman invitaion so that it slightl...   \n",
       "8   169270  moving headset across a surface until it falls...   \n",
       "9    66533                                folding paper towel   \n",
       "10   67698                            taking a slice of onion   \n",
       "11   60195                 pretending to turn hat upside down   \n",
       "12  187992         pretending to take bottle from window side   \n",
       "13  102808                            moving color pencils up   \n",
       "14  218043  twisting (wringing) shirt wet until water come...   \n",
       "15   82056                holding a pencil next to tissue box   \n",
       "16   10158                                   twisting a cloth   \n",
       "17  113667                    taking tissue out of tissue box   \n",
       "18  134982             dropping headset in front of tools bag   \n",
       "19   37055  throwing sunscreen in the air and letting it fall   \n",
       "20  199780  lifting up one end of card without letting it ...   \n",
       "21  122775                 moving notebook towards the camera   \n",
       "22  151369                            moving coconut wafer up   \n",
       "23  170911                      showing that tumbler is empty   \n",
       "24   25020                      dropping biscuit behind razor   \n",
       "25  116814                      throwing a stone against tree   \n",
       "26  122986                            holding pencil over box   \n",
       "27  216433  moving a pouch and a mobile phone closer to ea...   \n",
       "28   47323                    pushing iron from right to left   \n",
       "29   15409                           pushing coin so it spins   \n",
       "30  123183               covering a water bottle with a cloth   \n",
       "31  208007               putting match into empty pill bottle   \n",
       "32   63908                  showing that pen is inside bucket   \n",
       "33  156381                                   throwing headset   \n",
       "34  116989                   pulling knife from right to left   \n",
       "35  114072                    tipping disposable lighter over   \n",
       "36  203597  moving tomato across a surface until it falls ...   \n",
       "37   14991                          covering stone with paper   \n",
       "38  101299              pulling chopsticks from right to left   \n",
       "39  186594  poking a stack of tiles without the stack coll...   \n",
       "40  111183           letting marble roll along a flat surface   \n",
       "41  105168  showing 'enak' vanilla canned milk behind 'tig...   \n",
       "42  190776                     dropping compass tool onto box   \n",
       "43  181638   throwing a calculator in the air and catching it   \n",
       "44  140721                     throwing a ball against a sofa   \n",
       "45  145959                                    squeezing paper   \n",
       "46  201957                         pushing pencil off of book   \n",
       "47   20060           throwing card in the air and catching it   \n",
       "48  203403       moving polish and a box closer to each other   \n",
       "49  133878        pushing newspaper so that it slightly moves   \n",
       "\n",
       "                                             template  \\\n",
       "0             Holding [something] next to [something]   \n",
       "1              Spreading [something] onto [something]   \n",
       "2                    Putting [something] on a surface   \n",
       "3   Lifting up one end of [something], then lettin...   \n",
       "4                                 Holding [something]   \n",
       "5              Pushing [something] from right to left   \n",
       "6               Spilling [something] onto [something]   \n",
       "7       Pushing [something] so that it slightly moves   \n",
       "8   Moving [something] across a surface until it f...   \n",
       "9                                 Folding [something]   \n",
       "10   Taking [one of many similar things on the table]   \n",
       "11         Pretending to turn [something] upside down   \n",
       "12    Pretending to take [something] from [somewhere]   \n",
       "13                              Moving [something] up   \n",
       "14  Twisting (wringing) [something] wet until wate...   \n",
       "15            Holding [something] next to [something]   \n",
       "16                               Twisting [something]   \n",
       "17              Taking [something] out of [something]   \n",
       "18       Dropping [something] in front of [something]   \n",
       "19  Throwing [something] in the air and letting it...   \n",
       "20  Lifting up one end of [something] without lett...   \n",
       "21              Moving [something] towards the camera   \n",
       "22                              Moving [something] up   \n",
       "23                  Showing that [something] is empty   \n",
       "24            Dropping [something] behind [something]   \n",
       "25           Throwing [something] against [something]   \n",
       "26               Holding [something] over [something]   \n",
       "27  Moving [something] and [something] closer to e...   \n",
       "28             Pushing [something] from right to left   \n",
       "29                    Pushing [something] so it spins   \n",
       "30              Covering [something] with [something]   \n",
       "31               Putting [something] into [something]   \n",
       "32     Showing that [something] is inside [something]   \n",
       "33                               Throwing [something]   \n",
       "34             Pulling [something] from right to left   \n",
       "35                           Tipping [something] over   \n",
       "36  Moving [something] across a surface until it f...   \n",
       "37              Covering [something] with [something]   \n",
       "38             Pulling [something] from right to left   \n",
       "39  Poking a stack of [something] without the stac...   \n",
       "40      Letting [something] roll along a flat surface   \n",
       "41             Showing [something] behind [something]   \n",
       "42              Dropping [something] onto [something]   \n",
       "43    Throwing [something] in the air and catching it   \n",
       "44           Throwing [something] against [something]   \n",
       "45                              Squeezing [something]   \n",
       "46             Pushing [something] off of [something]   \n",
       "47    Throwing [something] in the air and catching it   \n",
       "48  Moving [something] and [something] closer to e...   \n",
       "49      Pushing [something] so that it slightly moves   \n",
       "\n",
       "                                         placeholders  \n",
       "0                      [potato, vicks vaporub bottle]  \n",
       "1                                  [margarine, bread]  \n",
       "2                                               [pen]  \n",
       "3                                            [bottle]  \n",
       "4                                              [bulb]  \n",
       "5                                      [strap camera]  \n",
       "6                                [mouthwash, roll on]  \n",
       "7                               [spiderman invitaion]  \n",
       "8                                           [headset]  \n",
       "9                                       [paper towel]  \n",
       "10                                 [a slice of onion]  \n",
       "11                                              [hat]  \n",
       "12                              [bottle, window side]  \n",
       "13                                    [color pencils]  \n",
       "14                                            [shirt]  \n",
       "15                             [a pencil, tissue box]  \n",
       "16                                          [a cloth]  \n",
       "17                               [tissue, tissue box]  \n",
       "18                               [headset, tools bag]  \n",
       "19                                        [sunscreen]  \n",
       "20                                             [card]  \n",
       "21                                         [notebook]  \n",
       "22                                    [coconut wafer]  \n",
       "23                                          [tumbler]  \n",
       "24                                   [biscuit, razor]  \n",
       "25                                    [a stone, tree]  \n",
       "26                                      [pencil, box]  \n",
       "27                          [a pouch, a mobile phone]  \n",
       "28                                             [iron]  \n",
       "29                                             [coin]  \n",
       "30                          [a water bottle, a cloth]  \n",
       "31                         [match, empty pill bottle]  \n",
       "32                                      [pen, bucket]  \n",
       "33                                          [headset]  \n",
       "34                                            [knife]  \n",
       "35                               [disposable lighter]  \n",
       "36                                           [tomato]  \n",
       "37                                     [stone, paper]  \n",
       "38                                       [chopsticks]  \n",
       "39                                            [tiles]  \n",
       "40                                           [marble]  \n",
       "41  ['enak' vanilla canned milk, 'tiga sapi' choco...  \n",
       "42                                [compass tool, box]  \n",
       "43                                     [a calculator]  \n",
       "44                                   [a ball, a sofa]  \n",
       "45                                            [paper]  \n",
       "46                                     [pencil, book]  \n",
       "47                                             [card]  \n",
       "48                                    [polish, a box]  \n",
       "49                                        [newspaper]  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_vids_id.head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(168913, 4)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_vids_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>template</th>\n",
       "      <th>placeholders</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>74225</td>\n",
       "      <td>spinning cube that quickly stops spinning</td>\n",
       "      <td>Spinning [something] that quickly stops spinning</td>\n",
       "      <td>[cube]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>116154</td>\n",
       "      <td>showing clay box on top of wallet</td>\n",
       "      <td>Showing [something] on top of [something]</td>\n",
       "      <td>[clay box, wallet]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>198186</td>\n",
       "      <td>wiping words off of a paper</td>\n",
       "      <td>Wiping [something] off of [something]</td>\n",
       "      <td>[words, a paper]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>137878</td>\n",
       "      <td>pushing scissors so that it falls off the table</td>\n",
       "      <td>Pushing [something] so that it falls off the t...</td>\n",
       "      <td>[scissors]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>151151</td>\n",
       "      <td>turning the camera left while filming wall mou...</td>\n",
       "      <td>Turning the camera left while filming [something]</td>\n",
       "      <td>[wall mounted fan]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id                                              label  \\\n",
       "0   74225          spinning cube that quickly stops spinning   \n",
       "1  116154                  showing clay box on top of wallet   \n",
       "2  198186                        wiping words off of a paper   \n",
       "3  137878    pushing scissors so that it falls off the table   \n",
       "4  151151  turning the camera left while filming wall mou...   \n",
       "\n",
       "                                            template        placeholders  \n",
       "0   Spinning [something] that quickly stops spinning              [cube]  \n",
       "1          Showing [something] on top of [something]  [clay box, wallet]  \n",
       "2              Wiping [something] off of [something]    [words, a paper]  \n",
       "3  Pushing [something] so that it falls off the t...          [scissors]  \n",
       "4  Turning the camera left while filming [something]  [wall mounted fan]  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_vids_id.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24777, 4)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_vids_id.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grouping the data based on following classes, grouping together all the data which belongs to one action\n",
    "# we will be using below 9 classes:\n",
    "classes = {'Dropping [something]' : [], \n",
    "           'Holding [something]' : [],\n",
    "           'Moving [something]' : [], \n",
    "           'Picking [something]' : [],\n",
    "           'Poking [something]' : [],\n",
    "           'Pouring [something]' : [],\n",
    "           'Putting [something]' : [],\n",
    "           'Showing [something]' : [], \n",
    "           'Tearing [something]' : []}\n",
    "\n",
    "actual_class = {'Dropping [something]' : 'Dropping_something', \n",
    "           'Holding [something]' : 'Holding_something',\n",
    "           'Moving [something]' : 'Moving_something', \n",
    "           'Picking [something]' : 'Picking_something',\n",
    "           'Poking [something]' : 'Poking_something',\n",
    "           'Pouring [something]' : 'Pouring_something',\n",
    "           'Putting [something]' : 'Putting_something',\n",
    "           'Showing [something]' : 'Showing_something', \n",
    "           'Tearing [something]' : 'Tearing_something'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# utility function to return the class name from the template name\n",
    "def get_class_name(template_name):\n",
    "    for key in classes:\n",
    "        if key in template_name:\n",
    "            return key\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# grouping the training data\n",
    "# iterating the pd dataframe\n",
    "for index, row in train_vids_id.iterrows():\n",
    "    row_class = get_class_name(row['template'])\n",
    "    if (row_class != None):\n",
    "        # print ('Template: ', row['template'], '\\tclass: ', row_class)\n",
    "        classes[row_class].append((row['id'], row['placeholders'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'shoe cleaning brush'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes['Dropping [something]'][100][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class:  Dropping [something] \t\tLength:  4680\n",
      "Class:  Holding [something] \t\tLength:  6894\n",
      "Class:  Moving [something] \t\tLength:  14877\n",
      "Class:  Picking [something] \t\tLength:  980\n",
      "Class:  Poking [something] \t\tLength:  4194\n",
      "Class:  Pouring [something] \t\tLength:  1700\n",
      "Class:  Putting [something] \t\tLength:  16731\n",
      "Class:  Showing [something] \t\tLength:  4141\n",
      "Class:  Tearing [something] \t\tLength:  3021\n"
     ]
    }
   ],
   "source": [
    "# printing the len of each class\n",
    "for key in classes:\n",
    "    print ('Class: ', key, '\\t\\tLength: ', len(classes[key]))\n",
    "\n",
    "    holding something : ~14k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we got our classes and it's count\n",
    "# let's start getting images as 0.5 fps and store them in their respective folders\n",
    "import cv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 1\n",
    "\n",
    "# function to extract frames and save it in respective class folder\n",
    "def getFrame(sec, vidcap, output_dir, class_name, obj, count):\n",
    "    vidcap.set(cv2.CAP_PROP_POS_MSEC, sec*1000)\n",
    "    hasFrames, image = vidcap.read()\n",
    "    if hasFrames:\n",
    "        if not os.path.exists(output_dir + actual_class[class_name]): \n",
    "            os.makedirs(output_dir + actual_class[class_name])\n",
    "        \n",
    "        os.chdir(output_dir + actual_class[class_name])\n",
    "        cv2.imwrite(actual_class[class_name].split('_')[0] + '_' + obj + '_' + str(count) + \".jpg\", image)     # save frame as JPG file\n",
    "    return hasFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# since we will be using different classes in the dict to generate images,\n",
    "# let's hope for now there are no overlapping of vids in different keys.\n",
    "# To make things faster, lets implement multi-processing in python so that\n",
    "# our code executes faster and utilizes more cpu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current class:  Dropping [something]\n"
     ]
    }
   ],
   "source": [
    "# run this cell if you want to simply generate images without multi-processing\n",
    "\n",
    "# directory where all the videos are saved\n",
    "video_dir = 'D:\\\\something-something-project\\\\data\\\\videos\\\\20bn-something-something-v2\\\\'\n",
    "# directory where images will be saved\n",
    "output_dir = 'D:\\\\something-something-project\\\\test-images\\\\'\n",
    "\n",
    "for key in classes:\n",
    "    global count\n",
    "    count = 1\n",
    "    print (\"Current class: \", key)\n",
    "    break_flag = False\n",
    "    for vid in classes[key]:\n",
    "        vidcap = cv2.VideoCapture(video_dir + str(vid[0]) + '.webm')\n",
    "        sec = 0\n",
    "        fps = 1\n",
    "        success = getFrame(sec, vidcap, output_dir, key, vid[1], count)\n",
    "        while success:\n",
    "            count = count + 1\n",
    "            # limiting the number of images (frames) per class to 5000\n",
    "#             if (count == 5000):\n",
    "#                 break_flag = True\n",
    "#                 break\n",
    "            sec = sec + fps\n",
    "            sec = round(sec, 2)\n",
    "            success = getFrame(sec, vidcap, output_dir, key, vid[1], count)\n",
    "#         if break_flag:\n",
    "#             break\n",
    "        break\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class:  Dropping [something] \t\tLength:  827\n",
      "Class:  Holding [something] \t\tLength:  1053\n",
      "Class:  Moving [something] \t\tLength:  1911\n",
      "Class:  Picking [something] \t\tLength:  199\n",
      "Class:  Poking [something] \t\tLength:  419\n",
      "Class:  Pouring [something] \t\tLength:  458\n",
      "Class:  Putting [something] \t\tLength:  1894\n",
      "Class:  Showing [something] \t\tLength:  697\n",
      "Class:  Tearing [something] \t\tLength:  898\n"
     ]
    }
   ],
   "source": [
    "# doing the same thing for validation set now\n",
    "\n",
    "# grouping the data based on following classes, grouping together all the data which belongs to one action\n",
    "# we will be using below 9 classes:\n",
    "validation_classes = {'Dropping [something]' : [], \n",
    "           'Holding [something]' : [],\n",
    "           'Moving [something]' : [], \n",
    "           'Picking [something]' : [],\n",
    "           'Poking [something]' : [],\n",
    "           'Pouring [something]' : [],\n",
    "           'Putting [something]' : [],\n",
    "           'Showing [something]' : [], \n",
    "           'Tearing [something]' : []}\n",
    "\n",
    "\n",
    "# utility function to return the class name from the template name\n",
    "def get_validation_class_name(template_name):\n",
    "    for key in validation_classes:\n",
    "        if key in template_name:\n",
    "            return key\n",
    "    return None\n",
    "\n",
    "\n",
    "# grouping the validation data\n",
    "# iterating the pd dataframe\n",
    "for index, row in validation_vids_id.iterrows():\n",
    "    row_class = get_validation_class_name(row['template'])\n",
    "    if (row_class != None):\n",
    "        # print ('Template: ', row['template'], '\\tclass: ', row_class)\n",
    "        validation_classes[row_class].append(row['id'])\n",
    "        \n",
    "# printing the len of each class\n",
    "for key in validation_classes:\n",
    "    print ('Class: ', key, '\\t\\tLength: ', len(validation_classes[key]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write the validation classes data into the disk\n",
    "count = 1\n",
    "\n",
    "# function to extract frames and save it in respective class folder\n",
    "def getValidationFrame(sec, vidcap, output_dir, class_name, count):\n",
    "    vidcap.set(cv2.CAP_PROP_POS_MSEC, sec*1000)\n",
    "    hasFrames, image = vidcap.read()\n",
    "    if hasFrames:\n",
    "        if not os.path.exists(output_dir + actual_class[class_name]): \n",
    "            os.makedirs(output_dir + actual_class[class_name])\n",
    "        \n",
    "        os.chdir(output_dir + actual_class[class_name])\n",
    "        cv2.imwrite(actual_class[class_name] + '_image_' + str(count) + \".jpg\", image)     # save frame as JPG file\n",
    "    return hasFrames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current class:  Dropping [something]\n",
      "Current class:  Holding [something]\n",
      "Current class:  Moving [something]\n",
      "Current class:  Picking [something]\n",
      "Current class:  Poking [something]\n",
      "Current class:  Pouring [something]\n",
      "Current class:  Putting [something]\n",
      "Current class:  Showing [something]\n",
      "Current class:  Tearing [something]\n"
     ]
    }
   ],
   "source": [
    "# directory where all the videos are saved\n",
    "video_dir = 'D:\\\\something-something-project\\\\data\\\\videos\\\\20bn-something-something-v2\\\\'\n",
    "# directory where images will be saved\n",
    "output_dir = 'D:\\\\something-something-project\\\\validation-images\\\\'\n",
    "\n",
    "for key in validation_classes:\n",
    "    global count\n",
    "    count = 1\n",
    "    print (\"Current class: \", key)\n",
    "    for vid in validation_classes[key]:\n",
    "        vidcap = cv2.VideoCapture(video_dir + str(vid) + '.webm')\n",
    "        sec = 0\n",
    "        fps = 0.5\n",
    "        success = getValidationFrame(sec, vidcap, output_dir, key, count)\n",
    "        while success:\n",
    "            count = count + 1\n",
    "            sec = sec + fps\n",
    "            sec = round(sec, 2)\n",
    "            success = getValidationFrame(sec, vidcap, output_dir, key, count)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dogs-vs-cats",
   "language": "python",
   "name": "dogs-vs-cats"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}