aula03-intro-pandas-part2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy as sp\n",
    "import pandas as pd\n",
    "\n",
    "import re\n",
    "\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Aula 03 &mdash; Introdução a Pandas - Parte 2\n",
    "\n",
    "Renato Vimieiro\n",
    "rv2 {em} cin.ufpe.br\n",
    "\n",
    "março 2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Resumo\n",
    "\n",
    "- Nessa aula continuaremos a explorar Pandas\n",
    "- Continuaremos a usar os dados do MovieLens como exemplo\n",
    "- Reutilizaremos os dados descritos na aula 02"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "usuarios = pd.read_csv(\n",
    "    \"http://files.grouplens.org/datasets/movielens/ml-100k/u.user\",\n",
    "    sep='|',header=None, names=[\"user_id\", \"age\", \"gender\", \"occupation\", \"zip_code\"])\n",
    "filmes = pd.read_csv(\n",
    "    \"http://files.grouplens.org/datasets/movielens/ml-100k/u.item\",\n",
    "    sep='|',header=None, names=[\"movie_id\", \"movie_title\",  \"release_date\", \"video_release_date\", \"IMDb_URL\", \n",
    "                                \"unknown\", \"Action\", \"Adventure\", \"Animation\", \n",
    "        \"Children\", \"Comedy\", \"Crime\", \"Documentary\", \"Drama\", \"Fantasy\",\"FilmNoir\", \"Horror\", \n",
    "                                \"Musical\", \"Mystery\", \"Romance\", \"SciFi\",\"Thriller\", \"War\", \"Western\"],\n",
    "encoding='latin1')\n",
    "avaliacoes = pd.read_csv(\n",
    "    \"http://files.grouplens.org/datasets/movielens/ml-100k/u.data\",\n",
    "    sep='\\t',header=None, names=[\"user_id\", \"movie_id\", \"rating\", \"timestamp\"])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Indexação e seleção de dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    24\n",
      "1    53\n",
      "2    23\n",
      "3    24\n",
      "4    33\n",
      "Name: age, dtype: int64\n",
      "0    24\n",
      "1    53\n",
      "2    23\n",
      "3    24\n",
      "4    33\n",
      "Name: age, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Selecionando uma coluna\n",
    "print(usuarios['age'].head())\n",
    "print(usuarios.age.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>53</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>33</td>\n",
       "      <td>F</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age gender\n",
       "0   24      M\n",
       "1   53      F\n",
       "2   23      M\n",
       "3   24      M\n",
       "4   33      F"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Selecionando multiplas colunas\n",
    "\n",
    "usuarios[['age','gender']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>62</td>\n",
       "      <td>257</td>\n",
       "      <td>2</td>\n",
       "      <td>879372434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>286</td>\n",
       "      <td>1014</td>\n",
       "      <td>5</td>\n",
       "      <td>879781125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>200</td>\n",
       "      <td>222</td>\n",
       "      <td>5</td>\n",
       "      <td>876042340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>210</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>891035994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>224</td>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>888104457</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  movie_id  rating  timestamp\n",
       "10       62       257       2  879372434\n",
       "11      286      1014       5  879781125\n",
       "12      200       222       5  876042340\n",
       "13      210        40       3  891035994\n",
       "14      224        29       3  888104457"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Selecionando linhas com slice\n",
    "avaliacoes[10:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unhashable type: 'slice'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-942554eebfd1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Nao e possivel combinar selecao de linha e coluna diretamente via []\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mavaliacoes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rating'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2137\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2138\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2139\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2140\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2141\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_getitem_column\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2144\u001b[0m         \u001b[0;31m# get column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2145\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2146\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2148\u001b[0m         \u001b[0;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   1838\u001b[0m         \u001b[0;34m\"\"\"Return the cached item, item represents a label indexer.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1839\u001b[0m         \u001b[0mcache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_item_cache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1840\u001b[0;31m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1841\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1842\u001b[0m             \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unhashable type: 'slice'"
     ]
    }
   ],
   "source": [
    "# Nao e possivel combinar selecao de linha e coluna diretamente via []\n",
    "avaliacoes[1:5,'rating']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10    2\n",
      "11    5\n",
      "12    5\n",
      "13    3\n",
      "14    3\n",
      "15    3\n",
      "Name: rating, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>IMDb_URL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         movie_title release_date  \\\n",
       "0   Toy Story (1995)  01-Jan-1995   \n",
       "1   GoldenEye (1995)  01-Jan-1995   \n",
       "2  Four Rooms (1995)  01-Jan-1995   \n",
       "3  Get Shorty (1995)  01-Jan-1995   \n",
       "4     Copycat (1995)  01-Jan-1995   \n",
       "\n",
       "                                            IMDb_URL  \n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...  \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...  \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...  \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...  \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# A selecao de porcoes especificas sao feitas atraves dos atributos\n",
    "# loc permite selecao por labels\n",
    "print(avaliacoes.loc[10:15,'rating'])\n",
    "\n",
    "# iloc permite a selecao por indice\n",
    "display(filmes.iloc[:5,[1,2,4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>42</td>\n",
       "      <td>M</td>\n",
       "      <td>executive</td>\n",
       "      <td>98101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>57</td>\n",
       "      <td>M</td>\n",
       "      <td>administrator</td>\n",
       "      <td>91344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>53</td>\n",
       "      <td>M</td>\n",
       "      <td>lawyer</td>\n",
       "      <td>90703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>47</td>\n",
       "      <td>M</td>\n",
       "      <td>educator</td>\n",
       "      <td>29206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>45</td>\n",
       "      <td>M</td>\n",
       "      <td>scientist</td>\n",
       "      <td>55106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  age gender     occupation zip_code\n",
       "5         6   42      M      executive    98101\n",
       "6         7   57      M  administrator    91344\n",
       "9        10   53      M         lawyer    90703\n",
       "12       13   47      M       educator    29206\n",
       "13       14   45      M      scientist    55106"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ainda podemos selecionar os dados com mascaras \n",
    "# booleanas como em NumPy\n",
    "usuarios[(usuarios.age > 40) & \n",
    "         ~(usuarios.occupation.isin(['none','other']))].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Exercício:\n",
    "======\n",
    "\n",
    "- Mostre a média de idade das mulheres cientistas\n",
    "- Quantos filmes de animação foram lançados em 1968"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     user_id  age gender occupation zip_code\n",
      "174      175   26      F  scientist    21911\n",
      "729      730   31      F  scientist    32114\n",
      "929      930   28      F  scientist    07310\n",
      "28.333333333333332\n"
     ]
    }
   ],
   "source": [
    "cientistas = usuarios[(usuarios.gender=='F') \n",
    "                      & (usuarios.occupation=='scientist')]\n",
    "print(cientistas.head())\n",
    "print(cientistas.age.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "print(filmes[(filmes.Animation==1)&\n",
    "             filmes.release_date.str.contains('1968')].shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Atribuições de valores\n",
    "\n",
    "- A indexação através de `loc` e `iloc` retorna uma visão do data frame\n",
    "- Essa visão pode ser modificada como fizemos em NumPy\n",
    "- Além disso, podemos também acrescentar novos dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "      <th>gender</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>175</td>\n",
       "      <td>-1</td>\n",
       "      <td>F</td>\n",
       "      <td>scientist</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>730</td>\n",
       "      <td>31</td>\n",
       "      <td>F</td>\n",
       "      <td>scientist</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>930</td>\n",
       "      <td>28</td>\n",
       "      <td>F</td>\n",
       "      <td>scientist</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  age gender occupation zip_code\n",
       "174      175   -1      F  scientist     None\n",
       "729      730   31      F  scientist     None\n",
       "929      930   28      F  scientist     None"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c2 = cientistas.copy()\n",
    "c2.iloc[0,1] = -1\n",
    "c2.loc[:,'zip_code'] = None\n",
    "c2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_title</th>\n",
       "      <th>release_date</th>\n",
       "      <th>release_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>1995</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         movie_title release_date release_year\n",
       "0   Toy Story (1995)  01-Jan-1995         1995\n",
       "1   GoldenEye (1995)  01-Jan-1995         1995\n",
       "2  Four Rooms (1995)  01-Jan-1995         1995\n",
       "3  Get Shorty (1995)  01-Jan-1995         1995\n",
       "4     Copycat (1995)  01-Jan-1995         1995"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filmes['release_year'] = filmes.release_date.apply(\n",
    "        lambda x: not x is np.nan and \n",
    "    re.search(\"\\d+\\-\\w+\\-(\\d+)\",str(x)).group(1) or None)\n",
    "filmes[['movie_title','release_date','release_year']].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Operações básicas\n",
    "\n",
    "- Pandas conta com várias funções pré-definidas para obter estatísticas e informações básicas dos dados\n",
    "- Ela conta também, como vimos anteriormente, com uma função `apply` que aplica uma função aos elementos\n",
    "- Seguem alguns exemplos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mediana de idade dos usuarios  31.0\n",
      "Heavyweights (1994)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4    34174\n",
       "3    27145\n",
       "5    21201\n",
       "2    11370\n",
       "1     6110\n",
       "Name: rating, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Mediana de idade dos usuarios \", usuarios.age.median())\n",
    "\n",
    "# piorAvaliacao = avaliacoes.rating.argmin()\n",
    "piorAvaliacao = avaliacoes.rating.values.argmin()\n",
    "print(filmes[filmes.movie_id == avaliacoes.movie_id.iloc[piorAvaliacao]].movie_title.iloc[0])\n",
    "\n",
    "avaliacoes.rating.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Concat, merge e joins\n",
    "\n",
    "- Vimos no exemplo anterior que pode ser necessário fazer cruzamento de dados em diferentes tabelas\n",
    "- Essa é uma situação muito comum em bancos de dados, onde tabelas são frequentemente cruzadas\n",
    "- Pandas dispõe de mecanismos similares para cruzamento de dados"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A0</td>\n",
       "      <td>B0</td>\n",
       "      <td>C0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A1</td>\n",
       "      <td>B1</td>\n",
       "      <td>C1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A2</td>\n",
       "      <td>B2</td>\n",
       "      <td>C2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A3</td>\n",
       "      <td>B3</td>\n",
       "      <td>C3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A    B   C\n",
       "0   A0   B0  C0\n",
       "1   A1   B1  C1\n",
       "2   A2   B2  C2\n",
       "3   A3   B3  C3\n",
       "4  NaN  NaN  C4"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A forma mais simples de juntar diferentes dados e concatenacao\n",
    "A = pd.Series([\"A{}\".format(a) for a in range(4)],\n",
    "              index=range(4), name=\"A\")\n",
    "B = pd.Series([\"B{}\".format(a) for a in range(4)],\n",
    "              index=range(4), name= \"B\")\n",
    "C = pd.Series([\"C{}\".format(a) for a in range(5)],\n",
    "              index=range(5), name= \"C\")\n",
    "pd.concat([A,B,C],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Merge \n",
    "\n",
    "- Uma opção mais [SQL-like](https://www.codeproject.com/KB/database/Visual_SQL_Joins/Visual_SQL_JOINS_orig.jpg) é merge\n",
    "- De acordo com a documentação essa função foi otimizada para desempenho\n",
    "- A sintaxe é\n",
    "\n",
    "```python\n",
    "pd.merge(left, right, how='inner', \n",
    "on=None, left_on=None, right_on=None,\n",
    "left_index=False, right_index=False, sort=True,\n",
    "suffixes=('_x', '_y'),copy=True,indicator=False)\n",
    "```         "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>movie_title</th>\n",
       "      <th>release_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>840</td>\n",
       "      <td>1674</td>\n",
       "      <td>4</td>\n",
       "      <td>891211682</td>\n",
       "      <td>Mamma Roma (1962)</td>\n",
       "      <td>1962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>655</td>\n",
       "      <td>1640</td>\n",
       "      <td>3</td>\n",
       "      <td>888474646</td>\n",
       "      <td>Eighth Day, The (1996)</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>655</td>\n",
       "      <td>1637</td>\n",
       "      <td>3</td>\n",
       "      <td>888984255</td>\n",
       "      <td>Girls Town (1996)</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>655</td>\n",
       "      <td>1630</td>\n",
       "      <td>3</td>\n",
       "      <td>887428735</td>\n",
       "      <td>Silence of the Palace, The (Saimt el Qusur) (1...</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>655</td>\n",
       "      <td>1641</td>\n",
       "      <td>3</td>\n",
       "      <td>887427810</td>\n",
       "      <td>Dadetown (1995)</td>\n",
       "      <td>1996</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  movie_id  rating  timestamp  \\\n",
       "99995      840      1674       4  891211682   \n",
       "99996      655      1640       3  888474646   \n",
       "99997      655      1637       3  888984255   \n",
       "99998      655      1630       3  887428735   \n",
       "99999      655      1641       3  887427810   \n",
       "\n",
       "                                             movie_title release_year  \n",
       "99995                                  Mamma Roma (1962)         1962  \n",
       "99996                             Eighth Day, The (1996)         1996  \n",
       "99997                                  Girls Town (1996)         1996  \n",
       "99998  Silence of the Palace, The (Saimt el Qusur) (1...         1996  \n",
       "99999                                    Dadetown (1995)         1996  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avaliacaoFilmes = pd.merge(avaliacoes,\n",
    "                           filmes[[\"movie_id\",\"movie_title\",\n",
    "                                   \"release_year\"]], \n",
    "                           on=\"movie_id\")\n",
    "avaliacaoFilmes.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Exercício:\n",
    "======\n",
    "\n",
    "- Calcule a média e desvio padrão das avaliações dadas aos filmes da década de 80 por programadores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.82152466367713, 1.0184489421764038)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avaliacaoFilmes.release_year.fillna(value=-1,inplace=True)\n",
    "avaliacaoFilmes['release_year'] = \\\n",
    "avaliacaoFilmes.release_year.astype(np.int)\n",
    "avalFilmUsu = pd.merge(avaliacaoFilmes,usuarios[[\"user_id\",\"occupation\"]], \n",
    "                       on=\"user_id\")\n",
    "selecao = avalFilmUsu[(1980 <= avalFilmUsu.release_year) & \n",
    "            (avalFilmUsu['release_year'] < 1990) \n",
    "            & (avalFilmUsu.occupation == 'programmer')]\n",
    "selecao.rating.mean(), selecao.rating.std() \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Split-apply-combine (1/3)\n",
    "\n",
    "- Um recurso muito útil de Pandas é o *group-by*\n",
    "- Podemos agrupar dados a partir de um critério para aplicar determinada função\n",
    "- Por exemplo, pode ser útil saber a média das avaliações por gênero do filme, ou faixa de idade\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Split-apply-combine (2/3)\n",
    "\n",
    "- O procedimento é executado da seguinte forma:\n",
    "    1. **Split**: dividir dados conforme um critério (e.g. coluna)\n",
    "    2. **Apply**: aplicar uma determinada função a cada um dos grupos\n",
    "    3. **Combine**: reunir resultados em uma única estrutura de dados"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Split-apply-combine (3/3)\n",
    "\n",
    "- A função que será aplicada aos grupos pode ter como objetivo:\n",
    "    - Sumarizar informações: computar estatísticas descritivas dos grupos (média, desvios, mínimo/máximo, contar valores, etc.)\n",
    "    - Transformar valores: normalizar dados nos grupos; gerar novos atributos; preencher dados ausentes\n",
    "    - Filtrar dados: eliminar grupos a partir de estatísticas computadas para o grupo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Exemplos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>occupation</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>administrator</th>\n",
       "      <td>430.949367</td>\n",
       "      <td>38.746835</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>artist</th>\n",
       "      <td>451.892857</td>\n",
       "      <td>31.392857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>doctor</th>\n",
       "      <td>533.714286</td>\n",
       "      <td>43.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>educator</th>\n",
       "      <td>466.905263</td>\n",
       "      <td>42.010526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>engineer</th>\n",
       "      <td>456.328358</td>\n",
       "      <td>36.388060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>entertainment</th>\n",
       "      <td>398.000000</td>\n",
       "      <td>29.222222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>executive</th>\n",
       "      <td>422.312500</td>\n",
       "      <td>38.718750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>healthcare</th>\n",
       "      <td>501.437500</td>\n",
       "      <td>41.562500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>homemaker</th>\n",
       "      <td>443.000000</td>\n",
       "      <td>32.571429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lawyer</th>\n",
       "      <td>359.083333</td>\n",
       "      <td>36.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>librarian</th>\n",
       "      <td>486.588235</td>\n",
       "      <td>40.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>marketing</th>\n",
       "      <td>437.807692</td>\n",
       "      <td>37.615385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>368.666667</td>\n",
       "      <td>26.555556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>542.733333</td>\n",
       "      <td>34.523810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>programmer</th>\n",
       "      <td>435.530303</td>\n",
       "      <td>33.121212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>retired</th>\n",
       "      <td>515.714286</td>\n",
       "      <td>63.071429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>salesman</th>\n",
       "      <td>494.916667</td>\n",
       "      <td>35.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scientist</th>\n",
       "      <td>465.129032</td>\n",
       "      <td>35.548387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>student</th>\n",
       "      <td>484.954082</td>\n",
       "      <td>22.081633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>technician</th>\n",
       "      <td>497.629630</td>\n",
       "      <td>33.148148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>writer</th>\n",
       "      <td>495.711111</td>\n",
       "      <td>36.311111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  user_id        age\n",
       "occupation                          \n",
       "administrator  430.949367  38.746835\n",
       "artist         451.892857  31.392857\n",
       "doctor         533.714286  43.571429\n",
       "educator       466.905263  42.010526\n",
       "engineer       456.328358  36.388060\n",
       "entertainment  398.000000  29.222222\n",
       "executive      422.312500  38.718750\n",
       "healthcare     501.437500  41.562500\n",
       "homemaker      443.000000  32.571429\n",
       "lawyer         359.083333  36.750000\n",
       "librarian      486.588235  40.000000\n",
       "marketing      437.807692  37.615385\n",
       "none           368.666667  26.555556\n",
       "other          542.733333  34.523810\n",
       "programmer     435.530303  33.121212\n",
       "retired        515.714286  63.071429\n",
       "salesman       494.916667  35.666667\n",
       "scientist      465.129032  35.548387\n",
       "student        484.954082  22.081633\n",
       "technician     497.629630  33.148148\n",
       "writer         495.711111  36.311111"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "usuarioPorProfissao = usuarios.groupby(usuarios.occupation,\n",
    "                                       as_index=True)\n",
    "#usuarioPorProfissao.age.mean()\n",
    "usuarioPorProfissao.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "count    1681.00000\n",
      "mean     1989.38608\n",
      "std        14.25358\n",
      "min      1922.00000\n",
      "25%      1993.00000\n",
      "50%      1995.00000\n",
      "75%      1996.00000\n",
      "max      1998.00000\n",
      "Name: release_year, dtype: float64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unknown</th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children</th>\n",
       "      <th>Comedy</th>\n",
       "      <th>Crime</th>\n",
       "      <th>Documentary</th>\n",
       "      <th>Drama</th>\n",
       "      <th>Fantasy</th>\n",
       "      <th>FilmNoir</th>\n",
       "      <th>Horror</th>\n",
       "      <th>Musical</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Romance</th>\n",
       "      <th>SciFi</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>release_year</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30s</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40s</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50s</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60s</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70s</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80s</th>\n",
       "      <td>0</td>\n",
       "      <td>33</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>53</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>15</td>\n",
       "      <td>17</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90s</th>\n",
       "      <td>0</td>\n",
       "      <td>202</td>\n",
       "      <td>93</td>\n",
       "      <td>26</td>\n",
       "      <td>88</td>\n",
       "      <td>410</td>\n",
       "      <td>89</td>\n",
       "      <td>46</td>\n",
       "      <td>591</td>\n",
       "      <td>18</td>\n",
       "      <td>7</td>\n",
       "      <td>57</td>\n",
       "      <td>25</td>\n",
       "      <td>41</td>\n",
       "      <td>195</td>\n",
       "      <td>70</td>\n",
       "      <td>208</td>\n",
       "      <td>42</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              unknown  Action  Adventure  Animation  Children  Comedy  Crime  \\\n",
       "release_year                                                                   \n",
       "30s                 0       1          3          3         5      14      2   \n",
       "40s                 0       0          3          3         5       4      4   \n",
       "50s                 0       3          6          1         6      15      3   \n",
       "60s                 0       3          3          3         7       9      1   \n",
       "70s                 1       9          9          1         6      20      4   \n",
       "80s                 0      33         18          5         5      33      6   \n",
       "90s                 0     202         93         26        88     410     89   \n",
       "\n",
       "              Documentary  Drama  Fantasy  FilmNoir  Horror  Musical  Mystery  \\\n",
       "release_year                                                                    \n",
       "30s                     0     11        0         1       2        7        1   \n",
       "40s                     0     16        0        10       1        5        9   \n",
       "50s                     0     22        1         1       2        6        3   \n",
       "60s                     0     15        0         2       3        4        3   \n",
       "70s                     0     17        1         1      12        4        2   \n",
       "80s                     4     53        2         2      15        5        2   \n",
       "90s                    46    591       18         7      57       25       41   \n",
       "\n",
       "              Romance  SciFi  Thriller  War  Western  \n",
       "release_year                                          \n",
       "30s                10      0         4    3        0  \n",
       "40s                 6      0         7    3        2  \n",
       "50s                10      4         8    3        4  \n",
       "60s                 4      5         5    5        4  \n",
       "70s                 6      7         2    5        1  \n",
       "80s                16     15        17   10        3  \n",
       "90s               195     70       208   42       13  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(filmes.release_year.dropna().astype(np.int).describe())\n",
    "#print(pd.cut(filmes.release_year.dropna().astype(np.int),bins=10).head())\n",
    "decada = pd.cut(filmes.release_year.fillna(value=0).astype(np.int),\n",
    "       bins=[1900,1940,1950,1960,1970,1980,1990,2000],\n",
    "       labels=['{}s'.format(a) for a in np.arange(30,100,10)])\n",
    "filmePorDecada = filmes.groupby(decada)\n",
    "filmePorDecada[filmes.columns[5:]].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Exercício:\n",
    "======\n",
    "- Mostre a quantidade relativa de filmes de cada gênero por década\n",
    "- Calcule a avaliação média de cada gênero por faixa etária"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "release_year\n",
      "30s      39\n",
      "40s      44\n",
      "50s      52\n",
      "60s      41\n",
      "70s      58\n",
      "80s     123\n",
      "90s    1324\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>release_year</th>\n",
       "      <th>30s</th>\n",
       "      <th>40s</th>\n",
       "      <th>50s</th>\n",
       "      <th>60s</th>\n",
       "      <th>70s</th>\n",
       "      <th>80s</th>\n",
       "      <th>90s</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>unknown</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Action</th>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.073171</td>\n",
       "      <td>0.155172</td>\n",
       "      <td>0.268293</td>\n",
       "      <td>0.152568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Adventure</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.068182</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.073171</td>\n",
       "      <td>0.155172</td>\n",
       "      <td>0.146341</td>\n",
       "      <td>0.070242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Animation</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.068182</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>0.073171</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.040650</td>\n",
       "      <td>0.019637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Children</th>\n",
       "      <td>0.128205</td>\n",
       "      <td>0.113636</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.170732</td>\n",
       "      <td>0.103448</td>\n",
       "      <td>0.040650</td>\n",
       "      <td>0.066465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Comedy</th>\n",
       "      <td>0.358974</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.288462</td>\n",
       "      <td>0.219512</td>\n",
       "      <td>0.344828</td>\n",
       "      <td>0.268293</td>\n",
       "      <td>0.309668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Crime</th>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.090909</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.024390</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.048780</td>\n",
       "      <td>0.067221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Documentary</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.032520</td>\n",
       "      <td>0.034743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Drama</th>\n",
       "      <td>0.282051</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.423077</td>\n",
       "      <td>0.365854</td>\n",
       "      <td>0.293103</td>\n",
       "      <td>0.430894</td>\n",
       "      <td>0.446375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fantasy</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.016260</td>\n",
       "      <td>0.013595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FilmNoir</th>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.227273</td>\n",
       "      <td>0.019231</td>\n",
       "      <td>0.048780</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.016260</td>\n",
       "      <td>0.005287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Horror</th>\n",
       "      <td>0.051282</td>\n",
       "      <td>0.022727</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.073171</td>\n",
       "      <td>0.206897</td>\n",
       "      <td>0.121951</td>\n",
       "      <td>0.043051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Musical</th>\n",
       "      <td>0.179487</td>\n",
       "      <td>0.113636</td>\n",
       "      <td>0.115385</td>\n",
       "      <td>0.097561</td>\n",
       "      <td>0.068966</td>\n",
       "      <td>0.040650</td>\n",
       "      <td>0.018882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mystery</th>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.204545</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.073171</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.016260</td>\n",
       "      <td>0.030967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Romance</th>\n",
       "      <td>0.256410</td>\n",
       "      <td>0.136364</td>\n",
       "      <td>0.192308</td>\n",
       "      <td>0.097561</td>\n",
       "      <td>0.103448</td>\n",
       "      <td>0.130081</td>\n",
       "      <td>0.147281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SciFi</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.121951</td>\n",
       "      <td>0.120690</td>\n",
       "      <td>0.121951</td>\n",
       "      <td>0.052870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Thriller</th>\n",
       "      <td>0.102564</td>\n",
       "      <td>0.159091</td>\n",
       "      <td>0.153846</td>\n",
       "      <td>0.121951</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.138211</td>\n",
       "      <td>0.157100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>War</th>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.068182</td>\n",
       "      <td>0.057692</td>\n",
       "      <td>0.121951</td>\n",
       "      <td>0.086207</td>\n",
       "      <td>0.081301</td>\n",
       "      <td>0.031722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Western</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045455</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.097561</td>\n",
       "      <td>0.017241</td>\n",
       "      <td>0.024390</td>\n",
       "      <td>0.009819</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "release_year       30s       40s       50s       60s       70s       80s  \\\n",
       "unknown       0.000000  0.000000  0.000000  0.000000  0.017241  0.000000   \n",
       "Action        0.025641  0.000000  0.057692  0.073171  0.155172  0.268293   \n",
       "Adventure     0.076923  0.068182  0.115385  0.073171  0.155172  0.146341   \n",
       "Animation     0.076923  0.068182  0.019231  0.073171  0.017241  0.040650   \n",
       "Children      0.128205  0.113636  0.115385  0.170732  0.103448  0.040650   \n",
       "Comedy        0.358974  0.090909  0.288462  0.219512  0.344828  0.268293   \n",
       "Crime         0.051282  0.090909  0.057692  0.024390  0.068966  0.048780   \n",
       "Documentary   0.000000  0.000000  0.000000  0.000000  0.000000  0.032520   \n",
       "Drama         0.282051  0.363636  0.423077  0.365854  0.293103  0.430894   \n",
       "Fantasy       0.000000  0.000000  0.019231  0.000000  0.017241  0.016260   \n",
       "FilmNoir      0.025641  0.227273  0.019231  0.048780  0.017241  0.016260   \n",
       "Horror        0.051282  0.022727  0.038462  0.073171  0.206897  0.121951   \n",
       "Musical       0.179487  0.113636  0.115385  0.097561  0.068966  0.040650   \n",
       "Mystery       0.025641  0.204545  0.057692  0.073171  0.034483  0.016260   \n",
       "Romance       0.256410  0.136364  0.192308  0.097561  0.103448  0.130081   \n",
       "SciFi         0.000000  0.000000  0.076923  0.121951  0.120690  0.121951   \n",
       "Thriller      0.102564  0.159091  0.153846  0.121951  0.034483  0.138211   \n",
       "War           0.076923  0.068182  0.057692  0.121951  0.086207  0.081301   \n",
       "Western       0.000000  0.045455  0.076923  0.097561  0.017241  0.024390   \n",
       "\n",
       "release_year       90s  \n",
       "unknown       0.000000  \n",
       "Action        0.152568  \n",
       "Adventure     0.070242  \n",
       "Animation     0.019637  \n",
       "Children      0.066465  \n",
       "Comedy        0.309668  \n",
       "Crime         0.067221  \n",
       "Documentary   0.034743  \n",
       "Drama         0.446375  \n",
       "Fantasy       0.013595  \n",
       "FilmNoir      0.005287  \n",
       "Horror        0.043051  \n",
       "Musical       0.018882  \n",
       "Mystery       0.030967  \n",
       "Romance       0.147281  \n",
       "SciFi         0.052870  \n",
       "Thriller      0.157100  \n",
       "War           0.031722  \n",
       "Western       0.009819  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b = filmePorDecada[filmes.columns[5:]].sum()\n",
    "sizes = filmePorDecada.size()\n",
    "print(sizes)\n",
    "b.T/sizes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}