Working with Data Part 3.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:ad04d77842c9af2104d84f87f26fb16f029a647ca401211664b8a71e10ad98e8"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "from pandas import DataFrame, Series"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Let's make a dframe\n",
      "dframe = DataFrame({'k1':['X','X','Y','Y','Z'],\n",
      "                    'k2':['alpha','beta','alpha','beta','alpha'],\n",
      "                    'dataset1':np.random.randn(5),\n",
      "                    'dataset2':np.random.randn(5)})\n",
      "\n",
      "#Show\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0.087483</td>\n",
        "      <td> 1.466237</td>\n",
        "      <td> X</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 0.327045</td>\n",
        "      <td> 0.324467</td>\n",
        "      <td> X</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 0.453402</td>\n",
        "      <td>-1.772477</td>\n",
        "      <td> Y</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>-0.805783</td>\n",
        "      <td>-0.045210</td>\n",
        "      <td> Y</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>-1.430822</td>\n",
        "      <td>-0.374149</td>\n",
        "      <td> Z</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "   dataset1  dataset2 k1     k2\n",
        "0  0.087483  1.466237  X  alpha\n",
        "1  0.327045  0.324467  X   beta\n",
        "2  0.453402 -1.772477  Y  alpha\n",
        "3 -0.805783 -0.045210  Y   beta\n",
        "4 -1.430822 -0.374149  Z  alpha"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now let's see how to use groupby\n",
      "\n",
      "#Lets grab the dataset1 column and group it by the k1 key\n",
      "group1 = dframe['dataset1'].groupby(dframe['k1'])\n",
      "\n",
      "#Show the groupby object\n",
      "group1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "<pandas.core.groupby.SeriesGroupBy object at 0x0000000009FDD780>"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now we can perform operations on this particular group\n",
      "group1.mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "k1\n",
        "X     0.207264\n",
        "Y    -0.176190\n",
        "Z    -1.430822\n",
        "Name: dataset1, dtype: float64"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can use group keys that are series as well\n",
      "\n",
      "#For example:\n",
      "\n",
      "#We'll make some arrays for use as keys\n",
      "cities = np.array(['NY','LA','LA','NY','NY'])\n",
      "month = np.array(['JAN','FEB','JAN','FEB','JAN'])\n",
      "\n",
      "#Now using the data from dataset1, group the means by city and month\n",
      "dframe['dataset1'].groupby([cities,month]).mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "LA  FEB    0.327045\n",
        "    JAN    0.453402\n",
        "NY  FEB   -0.805783\n",
        "    JAN   -0.671670\n",
        "Name: dataset1, dtype: float64"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# let's see the original dframe again.\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0.087483</td>\n",
        "      <td> 1.466237</td>\n",
        "      <td> X</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 0.327045</td>\n",
        "      <td> 0.324467</td>\n",
        "      <td> X</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 0.453402</td>\n",
        "      <td>-1.772477</td>\n",
        "      <td> Y</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>-0.805783</td>\n",
        "      <td>-0.045210</td>\n",
        "      <td> Y</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>-1.430822</td>\n",
        "      <td>-0.374149</td>\n",
        "      <td> Z</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "   dataset1  dataset2 k1     k2\n",
        "0  0.087483  1.466237  X  alpha\n",
        "1  0.327045  0.324467  X   beta\n",
        "2  0.453402 -1.772477  Y  alpha\n",
        "3 -0.805783 -0.045210  Y   beta\n",
        "4 -1.430822 -0.374149  Z  alpha"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# WE can also pass column names as group keys\n",
      "dframe.groupby('k1').mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>k1</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>X</th>\n",
        "      <td> 0.207264</td>\n",
        "      <td> 0.895352</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Y</th>\n",
        "      <td>-0.176190</td>\n",
        "      <td>-0.908844</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Z</th>\n",
        "      <td>-1.430822</td>\n",
        "      <td>-0.374149</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "    dataset1  dataset2\n",
        "k1                    \n",
        "X   0.207264  0.895352\n",
        "Y  -0.176190 -0.908844\n",
        "Z  -1.430822 -0.374149"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Or multiple column names\n",
      "dframe.groupby(['k1','k2']).mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">X</th>\n",
        "      <th>alpha</th>\n",
        "      <td> 0.087483</td>\n",
        "      <td> 1.466237</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>beta</th>\n",
        "      <td> 0.327045</td>\n",
        "      <td> 0.324467</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">Y</th>\n",
        "      <th>alpha</th>\n",
        "      <td> 0.453402</td>\n",
        "      <td>-1.772477</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>beta</th>\n",
        "      <td>-0.805783</td>\n",
        "      <td>-0.045210</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Z</th>\n",
        "      <th>alpha</th>\n",
        "      <td>-1.430822</td>\n",
        "      <td>-0.374149</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "          dataset1  dataset2\n",
        "k1 k2                       \n",
        "X  alpha  0.087483  1.466237\n",
        "   beta   0.327045  0.324467\n",
        "Y  alpha  0.453402 -1.772477\n",
        "   beta  -0.805783 -0.045210\n",
        "Z  alpha -1.430822 -0.374149"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Another useful groupby method is getting the group sizes\n",
      "dframe.groupby(['k1']).size()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "k1\n",
        "X     2\n",
        "Y     2\n",
        "Z     1\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also iterate over groups\n",
      "\n",
      "#For example:\n",
      "for name,group in dframe.groupby('k1'):\n",
      "    print \"This is the %s group\" %name\n",
      "    print group\n",
      "    print '\\n'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "This is the X group\n",
        "   dataset1  dataset2 k1     k2\n",
        "0  0.087483  1.466237  X  alpha\n",
        "1  0.327045  0.324467  X   beta\n",
        "\n",
        "\n",
        "This is the Y group\n",
        "   dataset1  dataset2 k1     k2\n",
        "2  0.453402 -1.772477  Y  alpha\n",
        "3 -0.805783 -0.045210  Y   beta\n",
        "\n",
        "\n",
        "This is the Z group\n",
        "   dataset1  dataset2 k1     k2\n",
        "4 -1.430822 -0.374149  Z  alpha\n",
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also iterate with multiple keys\n",
      "for (k1,k2) , group in dframe.groupby(['k1','k2']):\n",
      "    print \"Key1 = %s Key2 = %s\" %(k1,k2)\n",
      "    print group\n",
      "    print '\\n'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Key1 = X Key2 = alpha\n",
        "   dataset1  dataset2 k1     k2\n",
        "0  0.087483  1.466237  X  alpha\n",
        "\n",
        "\n",
        "Key1 = X Key2 = beta\n",
        "   dataset1  dataset2 k1    k2\n",
        "1  0.327045  0.324467  X  beta\n",
        "\n",
        "\n",
        "Key1 = Y Key2 = alpha\n",
        "   dataset1  dataset2 k1     k2\n",
        "2  0.453402 -1.772477  Y  alpha\n",
        "\n",
        "\n",
        "Key1 = Y Key2 = beta\n",
        "   dataset1  dataset2 k1    k2\n",
        "3 -0.805783  -0.04521  Y  beta\n",
        "\n",
        "\n",
        "Key1 = Z Key2 = alpha\n",
        "   dataset1  dataset2 k1     k2\n",
        "4 -1.430822 -0.374149  Z  alpha\n",
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# A possibly useful tactic is creating a dictionary of the data pieces \n",
      "group_dict = dict(list(dframe.groupby('k1')))\n",
      "\n",
      "#Show the group with X\n",
      "group_dict['X']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0.087483</td>\n",
        "      <td> 1.466237</td>\n",
        "      <td> X</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 0.327045</td>\n",
        "      <td> 0.324467</td>\n",
        "      <td> X</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "   dataset1  dataset2 k1     k2\n",
        "0  0.087483  1.466237  X  alpha\n",
        "1  0.327045  0.324467  X   beta"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "group_dict['Y']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>dataset1</th>\n",
        "      <th>dataset2</th>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 0.453402</td>\n",
        "      <td>-1.772477</td>\n",
        "      <td> Y</td>\n",
        "      <td> alpha</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>-0.805783</td>\n",
        "      <td>-0.045210</td>\n",
        "      <td> Y</td>\n",
        "      <td>  beta</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "   dataset1  dataset2 k1     k2\n",
        "2  0.453402 -1.772477  Y  alpha\n",
        "3 -0.805783 -0.045210  Y   beta"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We could have also chosen to do this with axis = 1\n",
      "\n",
      "# Let's creat a dictionary for dtypes of objects!\n",
      "group_dict_axis1 = dict(list(dframe.groupby(dframe.dtypes,axis=1)))\n",
      "\n",
      "#show\n",
      "group_dict_axis1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "{dtype('float64'):    dataset1  dataset2\n",
        " 0  0.087483  1.466237\n",
        " 1  0.327045  0.324467\n",
        " 2  0.453402 -1.772477\n",
        " 3 -0.805783 -0.045210\n",
        " 4 -1.430822 -0.374149, dtype('O'):   k1     k2\n",
        " 0  X  alpha\n",
        " 1  X   beta\n",
        " 2  Y  alpha\n",
        " 3  Y   beta\n",
        " 4  Z  alpha}"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# For example if we only wanted to group the dataset2 column with both sets of keys\n",
      "dataset2_group = dframe.groupby(['k1','k2'])[['dataset2']]\n",
      "\n",
      "dataset2_group.mean()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th>dataset2</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>k1</th>\n",
        "      <th>k2</th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">X</th>\n",
        "      <th>alpha</th>\n",
        "      <td> 1.466237</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>beta</th>\n",
        "      <td> 0.324467</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">Y</th>\n",
        "      <th>alpha</th>\n",
        "      <td>-1.772477</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>beta</th>\n",
        "      <td>-0.045210</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Z</th>\n",
        "      <th>alpha</th>\n",
        "      <td>-0.374149</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "          dataset2\n",
        "k1 k2             \n",
        "X  alpha  1.466237\n",
        "   beta   0.324467\n",
        "Y  alpha -1.772477\n",
        "   beta  -0.045210\n",
        "Z  alpha -0.374149"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Groupby on Dict and Series"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's make a Dframe\n",
      "\n",
      "animals = DataFrame(np.arange(16).reshape(4, 4),\n",
      "                   columns=['W', 'X', 'Y', 'Z'],\n",
      "                   index=['Dog', 'Cat', 'Bird', 'Mouse'])\n",
      "\n",
      "animals"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>W</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Dog</th>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Cat</th>\n",
        "      <td>  4</td>\n",
        "      <td>  5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Bird</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Mouse</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 26,
       "text": [
        "        W   X   Y   Z\n",
        "Dog     0   1   2   3\n",
        "Cat     4   5   6   7\n",
        "Bird    8   9  10  11\n",
        "Mouse  12  13  14  15"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets add some NAN values\n",
      "animals.ix[1, ['W', 'Y']] = np.nan \n",
      "\n",
      "#Show\n",
      "animals"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>W</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Dog</th>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Cat</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  5</td>\n",
        "      <td>NaN</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Bird</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Mouse</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "        W   X   Y   Z\n",
        "Dog     0   1   2   3\n",
        "Cat   NaN   5 NaN   7\n",
        "Bird    8   9  10  11\n",
        "Mouse  12  13  14  15"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's say I had a dictionary with ebhavior values in it\n",
      "behavior_map = {'W': 'good', 'X': 'bad', 'Y': 'good','Z': 'bad'}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now we can groupby using that mapping\n",
      "animal_col = animals.groupby(behavior_map, axis=1)\n",
      "\n",
      "# Show the sum accroding to the groupby with the mapping\n",
      "animal_col.sum()\n",
      "\n",
      "# For example [dog][good] = [dog][Y]+[dog][W]\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>bad</th>\n",
        "      <th>good</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Dog</th>\n",
        "      <td>  4</td>\n",
        "      <td>  2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Cat</th>\n",
        "      <td> 12</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Bird</th>\n",
        "      <td> 20</td>\n",
        "      <td> 18</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Mouse</th>\n",
        "      <td> 28</td>\n",
        "      <td> 26</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "       bad  good\n",
        "Dog      4     2\n",
        "Cat     12   NaN\n",
        "Bird    20    18\n",
        "Mouse   28    26"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's try it with a Series\n",
      "behav_series = Series(behavior_map)\n",
      "\n",
      "#Show\n",
      "behav_series"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "W    good\n",
        "X     bad\n",
        "Y    good\n",
        "Z     bad\n",
        "dtype: object"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's groupby the Series\n",
      "\n",
      "animals.groupby(behav_series, axis=1).count()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>bad</th>\n",
        "      <th>good</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Dog</th>\n",
        "      <td> 2</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Cat</th>\n",
        "      <td> 2</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Bird</th>\n",
        "      <td> 2</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Mouse</th>\n",
        "      <td> 2</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "       bad  good\n",
        "Dog      2     2\n",
        "Cat      2     0\n",
        "Bird     2     2\n",
        "Mouse    2     2"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets assume we wanted to group by the length of the animal names, we can pass the len function into groupby!\n",
      "\n",
      "# Show\n",
      "animals.groupby(len).sum()\n",
      "\n",
      "#Note the index is now number of letters in the animal name"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>W</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  0</td>\n",
        "      <td>  6</td>\n",
        "      <td>  2</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "    W   X   Y   Z\n",
        "3   0   6   2  10\n",
        "4   8   9  10  11\n",
        "5  12  13  14  15"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also mix functions with arrays,dicts, and Series for groupby methods\n",
      "\n",
      "# Set a list for keys\n",
      "keys = ['A', 'B', 'A', 'B']\n",
      "\n",
      "# Now groupby length of name and the keys to show max values\n",
      "animals.groupby([len, keys]).max()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th>W</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">3</th>\n",
        "      <th>A</th>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>B</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  5</td>\n",
        "      <td>NaN</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <th>A</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <th>B</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "      W   X   Y   Z\n",
        "3 A   0   1   2   3\n",
        "  B NaN   5 NaN   7\n",
        "4 A   8   9  10  11\n",
        "5 B  12  13  14  15"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also use groupby with hierarchaly index levels\n",
      "\n",
      "#Create a hierarchal column index\n",
      "hier_col = pd.MultiIndex.from_arrays([['NY','NY','NY','SF','SF'],[1,2,3,1,2]],names=['City','sub_value'])\n",
      "\n",
      "# Create a dframe with hierarchal index\n",
      "dframe_hr = DataFrame(np.arange(25).reshape(5,5),columns=hier_col)\n",
      "\n",
      "#Multiply values by 100 for clarity\n",
      "dframe_hr = dframe_hr*100\n",
      "\n",
      "#Show\n",
      "dframe_hr"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr>\n",
        "      <th>City</th>\n",
        "      <th colspan=\"3\" halign=\"left\">NY</th>\n",
        "      <th colspan=\"2\" halign=\"left\">SF</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sub_value</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>    0</td>\n",
        "      <td>  100</td>\n",
        "      <td>  200</td>\n",
        "      <td>  300</td>\n",
        "      <td>  400</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  500</td>\n",
        "      <td>  600</td>\n",
        "      <td>  700</td>\n",
        "      <td>  800</td>\n",
        "      <td>  900</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 1000</td>\n",
        "      <td> 1100</td>\n",
        "      <td> 1200</td>\n",
        "      <td> 1300</td>\n",
        "      <td> 1400</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 1500</td>\n",
        "      <td> 1600</td>\n",
        "      <td> 1700</td>\n",
        "      <td> 1800</td>\n",
        "      <td> 1900</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 2000</td>\n",
        "      <td> 2100</td>\n",
        "      <td> 2200</td>\n",
        "      <td> 2300</td>\n",
        "      <td> 2400</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "City         NY                SF      \n",
        "sub_value     1     2     3     1     2\n",
        "0             0   100   200   300   400\n",
        "1           500   600   700   800   900\n",
        "2          1000  1100  1200  1300  1400\n",
        "3          1500  1600  1700  1800  1900\n",
        "4          2000  2100  2200  2300  2400"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}