04_Working_with_data_2.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:b0bf2de2460d0ea422757404bc9eaebce254bb5351a3020f566fb0d93e592f97"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import numpy as np\n",
      "import pandas as pd\n",
      "from pandas import Series, DataFrame"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's make a dframe\n",
      "\n",
      "dframe1 = DataFrame({'key':['X','Z','Y','Z','X','X'],'data_set_1': np.arange(6)})\n",
      "\n",
      "#Show\n",
      "dframe1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> Z</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Y</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> Z</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> X</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> 5</td>\n",
        "      <td> X</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "   data_set_1 key\n",
        "0           0   X\n",
        "1           1   Z\n",
        "2           2   Y\n",
        "3           3   Z\n",
        "4           4   X\n",
        "5           5   X"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets make another dframe\n",
      "\n",
      "dframe2 = DataFrame({'key':['Q','Y','Z'],'data_set_2':[1,2,3]})\n",
      "\n",
      "#Show\n",
      "dframe2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_2</th>\n",
        "      <th>key</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td> Q</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 2</td>\n",
        "      <td> Y</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 3</td>\n",
        "      <td> Z</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "   data_set_2 key\n",
        "0           1   Q\n",
        "1           2   Y\n",
        "2           3   Z"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now we can use merge the dataframes, this is a \"many-to-one\" situation\n",
      "\n",
      "# Merge will automatically choose overlapping columns to merge on\n",
      "pd.merge(dframe1,dframe2)\n",
      "\n",
      "#Note no overlapping 'X's"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 3</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Y</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "   data_set_1 key  data_set_2\n",
        "0           1   Z           3\n",
        "1           3   Z           3\n",
        "2           2   Y           2"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We could have also specified which column to merge on\n",
      "pd.merge(dframe1,dframe2,on='key')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 3</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Y</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "   data_set_1 key  data_set_2\n",
        "0           1   Z           3\n",
        "1           3   Z           3\n",
        "2           2   Y           2"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can choose which DataFrame's keys to use, this will choose left (dframe1)\n",
      "pd.merge(dframe1,dframe2,on='key',how='left')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 4</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 5</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 1</td>\n",
        "      <td> Z</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 3</td>\n",
        "      <td> Z</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> 2</td>\n",
        "      <td> Y</td>\n",
        "      <td>  2</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "   data_set_1 key  data_set_2\n",
        "0           0   X         NaN\n",
        "1           4   X         NaN\n",
        "2           5   X         NaN\n",
        "3           1   Z           3\n",
        "4           3   Z           3\n",
        "5           2   Y           2"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Choosing the one on the right (dframe2)\n",
      "pd.merge(dframe1,dframe2,on='key',how='right')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  1</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  3</td>\n",
        "      <td> Z</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  2</td>\n",
        "      <td> Y</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>NaN</td>\n",
        "      <td> Q</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "   data_set_1 key  data_set_2\n",
        "0           1   Z           3\n",
        "1           3   Z           3\n",
        "2           2   Y           2\n",
        "3         NaN   Q           1"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Choosing the \"outer\" method selects the union of both keys\n",
      "pd.merge(dframe1,dframe2,on='key',how='outer')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_1</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  0</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  4</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  5</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  1</td>\n",
        "      <td> Z</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  3</td>\n",
        "      <td> Z</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td>  2</td>\n",
        "      <td> Y</td>\n",
        "      <td>  2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td>NaN</td>\n",
        "      <td> Q</td>\n",
        "      <td>  1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "   data_set_1 key  data_set_2\n",
        "0           0   X         NaN\n",
        "1           4   X         NaN\n",
        "2           5   X         NaN\n",
        "3           1   Z           3\n",
        "4           3   Z           3\n",
        "5           2   Y           2\n",
        "6         NaN   Q           1"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now we'll learn about a many to many merge\n",
      "\n",
      "# Nnote that these DataFrames contain more than one instance of the key in BOTH datasets\n",
      "\n",
      "dframe3 = DataFrame({'key': ['X', 'X', 'X', 'Y', 'Z', 'Z'],\n",
      "                 'data_set_3': range(6)})\n",
      "dframe4 = DataFrame({'key': ['Y', 'Y', 'X', 'X', 'Z'],\n",
      "                 'data_set_4': range(5)})\n",
      "\n",
      "#Show the merge\n",
      "pd.merge(dframe3, dframe4)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set_3</th>\n",
        "      <th>key</th>\n",
        "      <th>data_set_4</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 1</td>\n",
        "      <td> X</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 1</td>\n",
        "      <td> X</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 2</td>\n",
        "      <td> X</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> 2</td>\n",
        "      <td> X</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td> 3</td>\n",
        "      <td> Y</td>\n",
        "      <td> 0</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7</th>\n",
        "      <td> 3</td>\n",
        "      <td> Y</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8</th>\n",
        "      <td> 4</td>\n",
        "      <td> Z</td>\n",
        "      <td> 4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9</th>\n",
        "      <td> 5</td>\n",
        "      <td> Z</td>\n",
        "      <td> 4</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "   data_set_3 key  data_set_4\n",
        "0           0   X           2\n",
        "1           0   X           3\n",
        "2           1   X           2\n",
        "3           1   X           3\n",
        "4           2   X           2\n",
        "5           2   X           3\n",
        "6           3   Y           0\n",
        "7           3   Y           1\n",
        "8           4   Z           4\n",
        "9           5   Z           4"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also merge with multiple keys!\n",
      "\n",
      "# Dframe on left\n",
      "df_left = DataFrame({'key1': ['SF', 'SF', 'LA'],\n",
      "                  'key2': ['one', 'two', 'one'],\n",
      "                  'left_data': [10,20,30]})\n",
      "\n",
      "#Dframe on right\n",
      "df_right = DataFrame({'key1': ['SF', 'SF', 'LA', 'LA'],\n",
      "                   'key2': ['one', 'one', 'one', 'two'],\n",
      "                   'right_data': [40,50,60,70]})\n",
      "\n",
      "#Merge\n",
      "pd.merge(df_left, df_right, on=['key1', 'key2'], how='outer')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "      <th>left_data</th>\n",
        "      <th>right_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> 40</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> 50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> SF</td>\n",
        "      <td> two</td>\n",
        "      <td> 20</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> LA</td>\n",
        "      <td> one</td>\n",
        "      <td> 30</td>\n",
        "      <td> 60</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> LA</td>\n",
        "      <td> two</td>\n",
        "      <td>NaN</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "  key1 key2  left_data  right_data\n",
        "0   SF  one         10          40\n",
        "1   SF  one         10          50\n",
        "2   SF  two         20         NaN\n",
        "3   LA  one         30          60\n",
        "4   LA  two        NaN          70"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Note that the left and right DataFrames have overlapping key names (key1 and key2).\n",
      "# pandas automatically adds suffixes to them\n",
      "\n",
      "pd.merge(df_left,df_right,on='key1')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2_x</th>\n",
        "      <th>left_data</th>\n",
        "      <th>key2_y</th>\n",
        "      <th>right_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> one</td>\n",
        "      <td> 40</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> one</td>\n",
        "      <td> 50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> SF</td>\n",
        "      <td> two</td>\n",
        "      <td> 20</td>\n",
        "      <td> one</td>\n",
        "      <td> 40</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> SF</td>\n",
        "      <td> two</td>\n",
        "      <td> 20</td>\n",
        "      <td> one</td>\n",
        "      <td> 50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> LA</td>\n",
        "      <td> one</td>\n",
        "      <td> 30</td>\n",
        "      <td> one</td>\n",
        "      <td> 60</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> LA</td>\n",
        "      <td> one</td>\n",
        "      <td> 30</td>\n",
        "      <td> two</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "  key1 key2_x  left_data key2_y  right_data\n",
        "0   SF    one         10    one          40\n",
        "1   SF    one         10    one          50\n",
        "2   SF    two         20    one          40\n",
        "3   SF    two         20    one          50\n",
        "4   LA    one         30    one          60\n",
        "5   LA    one         30    two          70"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also specify what the suffix becomes\n",
      "pd.merge(df_left,df_right, on='key1',suffixes=('_lefty','_righty'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2_lefty</th>\n",
        "      <th>left_data</th>\n",
        "      <th>key2_righty</th>\n",
        "      <th>right_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> one</td>\n",
        "      <td> 40</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> SF</td>\n",
        "      <td> one</td>\n",
        "      <td> 10</td>\n",
        "      <td> one</td>\n",
        "      <td> 50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> SF</td>\n",
        "      <td> two</td>\n",
        "      <td> 20</td>\n",
        "      <td> one</td>\n",
        "      <td> 40</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> SF</td>\n",
        "      <td> two</td>\n",
        "      <td> 20</td>\n",
        "      <td> one</td>\n",
        "      <td> 50</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> LA</td>\n",
        "      <td> one</td>\n",
        "      <td> 30</td>\n",
        "      <td> one</td>\n",
        "      <td> 60</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> LA</td>\n",
        "      <td> one</td>\n",
        "      <td> 30</td>\n",
        "      <td> two</td>\n",
        "      <td> 70</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "  key1 key2_lefty  left_data key2_righty  right_data\n",
        "0   SF        one         10         one          40\n",
        "1   SF        one         10         one          50\n",
        "2   SF        two         20         one          40\n",
        "3   SF        two         20         one          50\n",
        "4   LA        one         30         one          60\n",
        "5   LA        one         30         two          70"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# For more info on merge parameters check out:\n",
      "url = 'http://pandas.pydata.org/pandas-docs/dev/generated/pandas.DataFrame.merge.html'\n",
      "\n",
      "# Next we'll learn how to merge on Index!"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#merge on index"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets get two dframes\n",
      "\n",
      "df_left = DataFrame({'key': ['X','Y','Z','X','Y'],\n",
      "                  'data': range(5)})\n",
      "df_right = DataFrame({'group_data': [10, 20]}, index=['X', 'Y'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df_left"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data</th>\n",
        "      <th>key</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> Y</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Z</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> X</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> Y</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "   data key\n",
        "0     0   X\n",
        "1     1   Y\n",
        "2     2   Z\n",
        "3     3   X\n",
        "4     4   Y"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df_right"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>group_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>X</th>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Y</th>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "   group_data\n",
        "X          10\n",
        "Y          20"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now merge, we'll use the key for the left Dframe, and the index for the right\n",
      "pd.merge(df_left,df_right,left_on='key',right_index=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data</th>\n",
        "      <th>key</th>\n",
        "      <th>group_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> X</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> Y</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> Y</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "   data key  group_data\n",
        "0     0   X          10\n",
        "3     3   X          10\n",
        "1     1   Y          20\n",
        "4     4   Y          20"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also get a union by using outer\n",
      "pd.merge(df_left,df_right,left_on='key',right_index=True,how='outer')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data</th>\n",
        "      <th>key</th>\n",
        "      <th>group_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> X</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> Y</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> Y</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Z</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "   data key  group_data\n",
        "0     0   X          10\n",
        "3     3   X          10\n",
        "1     1   Y          20\n",
        "4     4   Y          20\n",
        "2     2   Z         NaN"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df_left_hr = DataFrame({'key1': ['SF','SF','SF','LA','LA'],\n",
      "                   'key2': [10, 20, 30, 20, 30],\n",
      "                   'data_set': np.arange(5.)})\n",
      "df_right_hr = DataFrame(np.arange(10).reshape((5, 2)),\n",
      "                   index=[['LA','LA','SF','SF','SF'],\n",
      "                          [20, 10, 10, 10, 20]],\n",
      "                   columns=['col_1', 'col_2'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#SHOW\n",
      "df_left_hr"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set</th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> SF</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> SF</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> SF</td>\n",
        "      <td> 30</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> LA</td>\n",
        "      <td> 20</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> LA</td>\n",
        "      <td> 30</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "   data_set key1  key2\n",
        "0         0   SF    10\n",
        "1         1   SF    20\n",
        "2         2   SF    30\n",
        "3         3   LA    20\n",
        "4         4   LA    30"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Show, this has a index hierarchy\n",
      "df_right_hr"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th>col_1</th>\n",
        "      <th>col_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th rowspan=\"2\" valign=\"top\">LA</th>\n",
        "      <th>20</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th rowspan=\"3\" valign=\"top\">SF</th>\n",
        "      <th>10</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td> 6</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>20</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "       col_1  col_2\n",
        "LA 20      0      1\n",
        "   10      2      3\n",
        "SF 10      4      5\n",
        "   10      6      7\n",
        "   20      8      9"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now we can merge the left by using keys and the right by its index\n",
      "pd.merge(df_left_hr,df_right_hr,left_on=['key1','key2'],right_index=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set</th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "      <th>col_1</th>\n",
        "      <th>col_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> SF</td>\n",
        "      <td> 10</td>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> SF</td>\n",
        "      <td> 10</td>\n",
        "      <td> 6</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> SF</td>\n",
        "      <td> 20</td>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> LA</td>\n",
        "      <td> 20</td>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "   data_set key1  key2  col_1  col_2\n",
        "0         0   SF    10      4      5\n",
        "0         0   SF    10      6      7\n",
        "1         1   SF    20      8      9\n",
        "3         3   LA    20      0      1"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can alo keep a union by choosing 'outer' method\n",
      "pd.merge(df_left_hr,df_right_hr,left_on=['key1','key2'],right_index=True,how='outer')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data_set</th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "      <th>col_1</th>\n",
        "      <th>col_2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  0</td>\n",
        "      <td> SF</td>\n",
        "      <td> 10</td>\n",
        "      <td>  4</td>\n",
        "      <td>  5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  0</td>\n",
        "      <td> SF</td>\n",
        "      <td> 10</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  1</td>\n",
        "      <td> SF</td>\n",
        "      <td> 20</td>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  2</td>\n",
        "      <td> SF</td>\n",
        "      <td> 30</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  3</td>\n",
        "      <td> LA</td>\n",
        "      <td> 20</td>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  4</td>\n",
        "      <td> LA</td>\n",
        "      <td> 30</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>NaN</td>\n",
        "      <td> LA</td>\n",
        "      <td> 10</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "   data_set key1  key2  col_1  col_2\n",
        "0         0   SF    10      4      5\n",
        "0         0   SF    10      6      7\n",
        "1         1   SF    20      8      9\n",
        "2         2   SF    30    NaN    NaN\n",
        "3         3   LA    20      0      1\n",
        "4         4   LA    30    NaN    NaN\n",
        "4       NaN   LA    10      2      3"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# WE can also you .join()\n",
      "\n",
      "# Shown on our first two DataFrames\n",
      "df_left.join(df_right)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>data</th>\n",
        "      <th>key</th>\n",
        "      <th>group_data</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1</td>\n",
        "      <td> Y</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2</td>\n",
        "      <td> Z</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 3</td>\n",
        "      <td> X</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 4</td>\n",
        "      <td> Y</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "   data key  group_data\n",
        "0     0   X         NaN\n",
        "1     1   Y         NaN\n",
        "2     2   Z         NaN\n",
        "3     3   X         NaN\n",
        "4     4   Y         NaN"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# concatenate "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create a matrix \n",
      "arr1 = np.arange(9).reshape((3,3))\n",
      "arr1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 26,
       "text": [
        "array([[0, 1, 2],\n",
        "       [3, 4, 5],\n",
        "       [6, 7, 8]])"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Concatenate along axis 1\n",
      "np.concatenate([arr1,arr1],axis=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "array([[0, 1, 2, 0, 1, 2],\n",
        "       [3, 4, 5, 3, 4, 5],\n",
        "       [6, 7, 8, 6, 7, 8]])"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's see other axis options\n",
      "np.concatenate([arr1,arr1],axis=0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 28,
       "text": [
        "array([[0, 1, 2],\n",
        "       [3, 4, 5],\n",
        "       [6, 7, 8],\n",
        "       [0, 1, 2],\n",
        "       [3, 4, 5],\n",
        "       [6, 7, 8]])"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets create two Series with no overlap\n",
      "ser1 =  Series([0,1,2],index=['T','U','V'])\n",
      "\n",
      "ser2 = Series([3,4],index=['X','Y'])\n",
      "\n",
      "#Now let use concat (default is axis=0)\n",
      "pd.concat([ser1,ser2])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "T    0\n",
        "U    1\n",
        "V    2\n",
        "X    3\n",
        "Y    4\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now passing along another axis will produce a DataFrame\n",
      "pd.concat([ser1,ser2],axis=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>T</th>\n",
        "      <td>  0</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>U</th>\n",
        "      <td>  1</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>V</th>\n",
        "      <td>  2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>X</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Y</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  4</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "    0   1\n",
        "T   0 NaN\n",
        "U   1 NaN\n",
        "V   2 NaN\n",
        "X NaN   3\n",
        "Y NaN   4"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can specify which specific axes to be used\n",
      "pd.concat([ser1,ser2],axis=1,join_axes=[['U','V','Y']])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>U</th>\n",
        "      <td>  1</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>V</th>\n",
        "      <td>  2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Y</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  4</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "    0   1\n",
        "U   1 NaN\n",
        "V   2 NaN\n",
        "Y NaN   4"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets say we wanted to add markers.keys to the concatenation result\n",
      "\n",
      "# WE can do this with a hierarchical index\n",
      "pd.concat([ser1,ser2],keys=['cat1','cat2'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "cat1  T    0\n",
        "      U    1\n",
        "      V    2\n",
        "cat2  X    3\n",
        "      Y    4\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Along the axis=1 then these Keys become column headers\n",
      "pd.concat([ser1,ser2],axis=1,keys=['cat1','cat2'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>cat1</th>\n",
        "      <th>cat2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>T</th>\n",
        "      <td>  0</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>U</th>\n",
        "      <td>  1</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>V</th>\n",
        "      <td>  2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>X</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Y</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  4</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "   cat1  cat2\n",
        "T     0   NaN\n",
        "U     1   NaN\n",
        "V     2   NaN\n",
        "X   NaN     3\n",
        "Y   NaN     4"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lastly, everything works similarly in DataFrames\n",
      "\n",
      "dframe1 = DataFrame(np.random.randn(4,3), columns=['X', 'Y', 'Z'])\n",
      "dframe2 = DataFrame(np.random.randn(3, 3), columns=['Y', 'Q', 'X'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Concat on DataFrame\n",
      "pd.concat([dframe1,dframe2])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Q</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>      NaN</td>\n",
        "      <td> 0.480077</td>\n",
        "      <td> 1.013189</td>\n",
        "      <td> 0.749800</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>      NaN</td>\n",
        "      <td> 0.322282</td>\n",
        "      <td>-1.286649</td>\n",
        "      <td>-0.937506</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>      NaN</td>\n",
        "      <td>-0.926129</td>\n",
        "      <td>-0.034254</td>\n",
        "      <td> 1.178470</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>      NaN</td>\n",
        "      <td>-0.107852</td>\n",
        "      <td>-1.899284</td>\n",
        "      <td>-0.582775</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>-3.072621</td>\n",
        "      <td>-1.431877</td>\n",
        "      <td> 0.013053</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1.027817</td>\n",
        "      <td>-0.685337</td>\n",
        "      <td> 1.435144</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>-0.139405</td>\n",
        "      <td> 1.600993</td>\n",
        "      <td> 0.436784</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 35,
       "text": [
        "          Q         X         Y         Z\n",
        "0       NaN  0.480077  1.013189  0.749800\n",
        "1       NaN  0.322282 -1.286649 -0.937506\n",
        "2       NaN -0.926129 -0.034254  1.178470\n",
        "3       NaN -0.107852 -1.899284 -0.582775\n",
        "0 -3.072621 -1.431877  0.013053       NaN\n",
        "1  1.027817 -0.685337  1.435144       NaN\n",
        "2 -0.139405  1.600993  0.436784       NaN"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#If we dont care about the index info and just awnt to make a complete DataFrame, just use ignore_index\n",
      "pd.concat([dframe1,dframe2],ignore_index=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Q</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>      NaN</td>\n",
        "      <td> 0.480077</td>\n",
        "      <td> 1.013189</td>\n",
        "      <td> 0.749800</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>      NaN</td>\n",
        "      <td> 0.322282</td>\n",
        "      <td>-1.286649</td>\n",
        "      <td>-0.937506</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>      NaN</td>\n",
        "      <td>-0.926129</td>\n",
        "      <td>-0.034254</td>\n",
        "      <td> 1.178470</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>      NaN</td>\n",
        "      <td>-0.107852</td>\n",
        "      <td>-1.899284</td>\n",
        "      <td>-0.582775</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>-3.072621</td>\n",
        "      <td>-1.431877</td>\n",
        "      <td> 0.013053</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5</th>\n",
        "      <td> 1.027817</td>\n",
        "      <td>-0.685337</td>\n",
        "      <td> 1.435144</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6</th>\n",
        "      <td>-0.139405</td>\n",
        "      <td> 1.600993</td>\n",
        "      <td> 0.436784</td>\n",
        "      <td>      NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "          Q         X         Y         Z\n",
        "0       NaN  0.480077  1.013189  0.749800\n",
        "1       NaN  0.322282 -1.286649 -0.937506\n",
        "2       NaN -0.926129 -0.034254  1.178470\n",
        "3       NaN -0.107852 -1.899284 -0.582775\n",
        "4 -3.072621 -1.431877  0.013053       NaN\n",
        "5  1.027817 -0.685337  1.435144       NaN\n",
        "6 -0.139405  1.600993  0.436784       NaN"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#For more info in documentation:\n",
      "url='http://pandas.pydata.org/pandas-docs/dev/generated/pandas.tools.merge.concat.html'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 37
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "#combining dataframe"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lets make some Series to work with\n",
      "\n",
      "#First Series\n",
      "ser1 = Series([2,np.nan,4,np.nan,6,np.nan],\n",
      "           index=['Q','R','S','T','U','V'])\n",
      "\n",
      "#Second Series (based off length of ser1)\n",
      "ser2 = Series(np.arange(len(ser1), dtype=np.float64),\n",
      "           index=['Q','R','S','T','U','V'])\n",
      "\n",
      "ser2[-1] = np.nan"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ser1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 39,
       "text": [
        "Q     2\n",
        "R   NaN\n",
        "S     4\n",
        "T   NaN\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ser2"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 40,
       "text": [
        "Q     0\n",
        "R     1\n",
        "S     2\n",
        "T     3\n",
        "U     4\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's get a series where the value of ser1 is chosen if ser2 is NAN,otherwise let the value be ser1\n",
      "Series(np.where(pd.isnull(ser1),ser2,ser1),index=ser1.index)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 41,
       "text": [
        "Q     2\n",
        "R     1\n",
        "S     4\n",
        "T     3\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now we can do the same thing simply by using combine_first with pandas\n",
      "ser1.combine_first(ser2)\n",
      "\n",
      "#This combines the Series values, choosing the values of the calling Series first, unless its a NAN"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 42,
       "text": [
        "Q     2\n",
        "R     1\n",
        "S     4\n",
        "T     3\n",
        "U     6\n",
        "V   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lets make some \n",
      "dframe_odds = DataFrame({'X': [1., np.nan, 3., np.nan],\n",
      "                     'Y': [np.nan, 5., np.nan, 7.],\n",
      "                     'Z': [np.nan, 9., np.nan, 11.]})\n",
      "dframe_evens = DataFrame({'X': [2., 4., np.nan, 6., 8.],\n",
      "                     'Y': [np.nan, 10., 12., 14., 16.]})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 43
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe_odds"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  1</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  5</td>\n",
        "      <td>  9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  3</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>NaN</td>\n",
        "      <td>  7</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 44,
       "text": [
        "    X   Y   Z\n",
        "0   1 NaN NaN\n",
        "1 NaN   5   9\n",
        "2   3 NaN NaN\n",
        "3 NaN   7  11"
       ]
      }
     ],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe_evens"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  4</td>\n",
        "      <td> 10</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>NaN</td>\n",
        "      <td> 12</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td>  6</td>\n",
        "      <td> 14</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>  8</td>\n",
        "      <td> 16</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 45,
       "text": [
        "    X   Y\n",
        "0   2 NaN\n",
        "1   4  10\n",
        "2 NaN  12\n",
        "3   6  14\n",
        "4   8  16"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets combine using odds values first, unless theres a NAN, then put the evens values\n",
      "dframe_odds.combine_first(dframe_evens)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 1</td>\n",
        "      <td>NaN</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 4</td>\n",
        "      <td>  5</td>\n",
        "      <td>  9</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 3</td>\n",
        "      <td> 12</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 6</td>\n",
        "      <td>  7</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> 8</td>\n",
        "      <td> 16</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 46,
       "text": [
        "   X   Y   Z\n",
        "0  1 NaN NaN\n",
        "1  4   5   9\n",
        "2  3  12 NaN\n",
        "3  6   7  11\n",
        "4  8  16 NaN"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# dataframe reshaping"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Let's see how stack and unstack work\n",
      "\n",
      "# Create DataFrame\n",
      "dframe1 = DataFrame(np.arange(8).reshape((2, 4)),\n",
      "                 index=pd.Index(['LA', 'SF'], name='city'),\n",
      "                 columns=pd.Index(['A', 'B', 'C','D'], name='letter'))\n",
      "#Show\n",
      "dframe1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>letter</th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>city</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>LA</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>SF</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td> 6</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "letter  A  B  C  D\n",
        "city              \n",
        "LA      0  1  2  3\n",
        "SF      4  5  6  7"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Use stack to pivot the columns into the rows\n",
      "dframe_st = dframe1.stack()\n",
      "\n",
      "#Show\n",
      "dframe_st"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "city  letter\n",
        "LA    A         0\n",
        "      B         1\n",
        "      C         2\n",
        "      D         3\n",
        "SF    A         4\n",
        "      B         5\n",
        "      C         6\n",
        "      D         7\n",
        "dtype: int32"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe_st[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "0"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#We can always rearrange back into a DataFrame\n",
      "dframe_st.unstack()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>letter</th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>city</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>LA</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>SF</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td> 6</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "letter  A  B  C  D\n",
        "city              \n",
        "LA      0  1  2  3\n",
        "SF      4  5  6  7"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#We can choose which level to unstack by\n",
      "dframe_st.unstack(0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>city</th>\n",
        "      <th>LA</th>\n",
        "      <th>SF</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>letter</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>A</th>\n",
        "      <td> 0</td>\n",
        "      <td> 4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>B</th>\n",
        "      <td> 1</td>\n",
        "      <td> 5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>C</th>\n",
        "      <td> 2</td>\n",
        "      <td> 6</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>D</th>\n",
        "      <td> 3</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "city    LA  SF\n",
        "letter        \n",
        "A        0   4\n",
        "B        1   5\n",
        "C        2   6\n",
        "D        3   7"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Also by which name to unstack by\n",
      "dframe_st.unstack('letter')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>letter</th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>city</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>LA</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 2</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>SF</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td> 6</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "letter  A  B  C  D\n",
        "city              \n",
        "LA      0  1  2  3\n",
        "SF      4  5  6  7"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Also by which name to unstack by\n",
      "dframe_st.unstack('city')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>city</th>\n",
        "      <th>LA</th>\n",
        "      <th>SF</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>letter</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>A</th>\n",
        "      <td> 0</td>\n",
        "      <td> 4</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>B</th>\n",
        "      <td> 1</td>\n",
        "      <td> 5</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>C</th>\n",
        "      <td> 2</td>\n",
        "      <td> 6</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>D</th>\n",
        "      <td> 3</td>\n",
        "      <td> 7</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "city    LA  SF\n",
        "letter        \n",
        "A        0   4\n",
        "B        1   5\n",
        "C        2   6\n",
        "D        3   7"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's see how stack and unstack handle NAN\n",
      "\n",
      "#Make two series\n",
      "ser1 = Series([0, 1, 2], index=['Q', 'X', 'Y'])\n",
      "ser2 = Series([4, 5, 6], index=['X', 'Y', 'Z'])\n",
      "\n",
      "#Concat to make a dframe\n",
      "dframe = pd.concat([ser1, ser2], keys=['Alpha', 'Beta'])\n",
      "\n",
      "# Unstack resulting DataFrame\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "Alpha  Q    0\n",
        "       X    1\n",
        "       Y    2\n",
        "Beta   X    4\n",
        "       Y    5\n",
        "       Z    6\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe.unstack()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>Q</th>\n",
        "      <th>X</th>\n",
        "      <th>Y</th>\n",
        "      <th>Z</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Alpha</th>\n",
        "      <td>  0</td>\n",
        "      <td> 1</td>\n",
        "      <td> 2</td>\n",
        "      <td>NaN</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Beta</th>\n",
        "      <td>NaN</td>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "        Q  X  Y   Z\n",
        "Alpha   0  1  2 NaN\n",
        "Beta  NaN  4  5   6"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now stack will filter out NAN by default\n",
      "dframe.unstack().stack()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "Alpha  Q    0\n",
        "       X    1\n",
        "       Y    2\n",
        "Beta   X    4\n",
        "       Y    5\n",
        "       Z    6\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# IF we dont want this we can set it to False\n",
      "dframe.unstack().stack(dropna=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "Alpha  Q     0\n",
        "       X     1\n",
        "       Y     2\n",
        "       Z   NaN\n",
        "Beta   Q   NaN\n",
        "       X     4\n",
        "       Y     5\n",
        "       Z     6\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Pivot"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets create some data to play with:\n",
      "\n",
      "# Note: It is not necessary to understand how this dataset was made to understand this Lecture.\n",
      "\n",
      "#import pandas testing utility\n",
      "import pandas.util.testing as tm; tm.N = 3\n",
      "\n",
      "#Create a unpivoted function\n",
      "def unpivot(frame):\n",
      "    N, K = frame.shape\n",
      "    \n",
      "    data = {'value' : frame.values.ravel('F'),\n",
      "            'variable' : np.asarray(frame.columns).repeat(N),\n",
      "            'date' : np.tile(np.asarray(frame.index), K)}\n",
      "    \n",
      "    # Return the DataFrame\n",
      "    return DataFrame(data, columns=['date', 'variable', 'value'])\n",
      "\n",
      "#Set the DataFrame we'll be using\n",
      "dframe = unpivot(tm.makeTimeDataFrame())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>date</th>\n",
        "      <th>variable</th>\n",
        "      <th>value</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0 </th>\n",
        "      <td>2000-01-03</td>\n",
        "      <td> A</td>\n",
        "      <td> 1.338805</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1 </th>\n",
        "      <td>2000-01-04</td>\n",
        "      <td> A</td>\n",
        "      <td> 0.405477</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2 </th>\n",
        "      <td>2000-01-05</td>\n",
        "      <td> A</td>\n",
        "      <td>-2.822585</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3 </th>\n",
        "      <td>2000-01-03</td>\n",
        "      <td> B</td>\n",
        "      <td> 0.163368</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4 </th>\n",
        "      <td>2000-01-04</td>\n",
        "      <td> B</td>\n",
        "      <td>-0.382342</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>5 </th>\n",
        "      <td>2000-01-05</td>\n",
        "      <td> B</td>\n",
        "      <td>-0.560083</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>6 </th>\n",
        "      <td>2000-01-03</td>\n",
        "      <td> C</td>\n",
        "      <td> 1.270668</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>7 </th>\n",
        "      <td>2000-01-04</td>\n",
        "      <td> C</td>\n",
        "      <td> 1.149571</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>8 </th>\n",
        "      <td>2000-01-05</td>\n",
        "      <td> C</td>\n",
        "      <td>-0.079408</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>9 </th>\n",
        "      <td>2000-01-03</td>\n",
        "      <td> D</td>\n",
        "      <td> 1.122493</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>10</th>\n",
        "      <td>2000-01-04</td>\n",
        "      <td> D</td>\n",
        "      <td> 0.420219</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>11</th>\n",
        "      <td>2000-01-05</td>\n",
        "      <td> D</td>\n",
        "      <td>-0.991808</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "         date variable     value\n",
        "0  2000-01-03        A  1.338805\n",
        "1  2000-01-04        A  0.405477\n",
        "2  2000-01-05        A -2.822585\n",
        "3  2000-01-03        B  0.163368\n",
        "4  2000-01-04        B -0.382342\n",
        "5  2000-01-05        B -0.560083\n",
        "6  2000-01-03        C  1.270668\n",
        "7  2000-01-04        C  1.149571\n",
        "8  2000-01-05        C -0.079408\n",
        "9  2000-01-03        D  1.122493\n",
        "10 2000-01-04        D  0.420219\n",
        "11 2000-01-05        D -0.991808"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now let's pivot the data\n",
      "\n",
      "# First two value spassed are teh row and column indexes, then finally an optional fill value\n",
      "dframe_piv = dframe.pivot('date','variable','value')\n",
      "\n",
      "#Show\n",
      "dframe_piv"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th>variable</th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>date</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>2000-01-03</th>\n",
        "      <td> 1.338805</td>\n",
        "      <td> 0.163368</td>\n",
        "      <td> 1.270668</td>\n",
        "      <td> 1.122493</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2000-01-04</th>\n",
        "      <td> 0.405477</td>\n",
        "      <td>-0.382342</td>\n",
        "      <td> 1.149571</td>\n",
        "      <td> 0.420219</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2000-01-05</th>\n",
        "      <td>-2.822585</td>\n",
        "      <td>-0.560083</td>\n",
        "      <td>-0.079408</td>\n",
        "      <td>-0.991808</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 20,
       "text": [
        "variable           A         B         C         D\n",
        "date                                              \n",
        "2000-01-03  1.338805  0.163368  1.270668  1.122493\n",
        "2000-01-04  0.405477 -0.382342  1.149571  0.420219\n",
        "2000-01-05 -2.822585 -0.560083 -0.079408 -0.991808"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Duplicates in Dataframe"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Lets get a dataframe with duplicates\n",
      "\n",
      "dframe = DataFrame({'key1': ['A'] * 2 + ['B'] * 3,\n",
      "                  'key2': [2, 2, 2, 3, 3]})\n",
      "\n",
      "#Show\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> A</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> A</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> B</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> B</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> B</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "  key1  key2\n",
        "0    A     2\n",
        "1    A     2\n",
        "2    B     2\n",
        "3    B     3\n",
        "4    B     3"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#We can use duplicated to find duplicates\n",
      "dframe.duplicated()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 22,
       "text": [
        "0    False\n",
        "1     True\n",
        "2    False\n",
        "3    False\n",
        "4     True\n",
        "dtype: bool"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also drop duplicates like this:\n",
      "dframe.drop_duplicates()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> A</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> B</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> B</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "  key1  key2\n",
        "0    A     2\n",
        "2    B     2\n",
        "3    B     3"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#You can filter which duplicates to drop by a single column\n",
      "dframe.drop_duplicates(['key1'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> A</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> B</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "  key1  key2\n",
        "0    A     2\n",
        "2    B     2"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#By default the first value was taken for the duplicates, we can also take the last value instead\n",
      "dframe.drop_duplicates(['key1'],take_last=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>key1</th>\n",
        "      <th>key2</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> A</td>\n",
        "      <td> 2</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td> B</td>\n",
        "      <td> 3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 26,
       "text": [
        "  key1  key2\n",
        "1    A     2\n",
        "4    B     3"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Mapping"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's create a dframe to work with (Highest elevation cities in USA)\n",
      "dframe = DataFrame({'city':['Alma','Brian Head','Fox Park'],\n",
      "                    'altitude':[3158,3000,2762]})\n",
      "\n",
      "#Show\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>altitude</th>\n",
        "      <th>city</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 3158</td>\n",
        "      <td>       Alma</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 3000</td>\n",
        "      <td> Brian Head</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2762</td>\n",
        "      <td>   Fox Park</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "   altitude        city\n",
        "0      3158        Alma\n",
        "1      3000  Brian Head\n",
        "2      2762    Fox Park"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now let's say we wanted to add a column for the States, we can do that with a mapping.\n",
      "state_map={'Alma':'Colorado','Brian Head':'Utah','Fox Park':'Wyoming'}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now we can map that data to our current dframe\n",
      "dframe['state'] = dframe['city'].map(state_map)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Show result\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>altitude</th>\n",
        "      <th>city</th>\n",
        "      <th>state</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 3158</td>\n",
        "      <td>       Alma</td>\n",
        "      <td> Colorado</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 3000</td>\n",
        "      <td> Brian Head</td>\n",
        "      <td>     Utah</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 2762</td>\n",
        "      <td>   Fox Park</td>\n",
        "      <td>  Wyoming</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 30,
       "text": [
        "   altitude        city     state\n",
        "0      3158        Alma  Colorado\n",
        "1      3000  Brian Head      Utah\n",
        "2      2762    Fox Park   Wyoming"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Replace"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets make  Series\n",
      "ser1 = Series([1,2,3,4,1,2,3,4])\n",
      "#Show\n",
      "ser1"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 31,
       "text": [
        "0    1\n",
        "1    2\n",
        "2    3\n",
        "3    4\n",
        "4    1\n",
        "5    2\n",
        "6    3\n",
        "7    4\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Using replace we can select --> .replace(value to be replaced, new_value)\n",
      "ser1.replace(1,np.nan)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "0   NaN\n",
        "1     2\n",
        "2     3\n",
        "3     4\n",
        "4   NaN\n",
        "5     2\n",
        "6     3\n",
        "7     4\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Can also input lists\n",
      "ser1.replace([1,4],[100,400])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "0    100\n",
        "1      2\n",
        "2      3\n",
        "3    400\n",
        "4    100\n",
        "5      2\n",
        "6      3\n",
        "7    400\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Can also input dictionary\n",
      "ser1.replace({4:np.nan})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 34,
       "text": [
        "0     1\n",
        "1     2\n",
        "2     3\n",
        "3   NaN\n",
        "4     1\n",
        "5     2\n",
        "6     3\n",
        "7   NaN\n",
        "dtype: float64"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Rename Index"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Making a DataFrame\n",
      "dframe= DataFrame(np.arange(12).reshape((3, 4)),\n",
      "                 index=['NY', 'LA', 'SF'],\n",
      "                 columns=['A', 'B', 'C', 'D'])\n",
      "\n",
      "#Show\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>NY</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>LA</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>SF</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 35,
       "text": [
        "    A  B   C   D\n",
        "NY  0  1   2   3\n",
        "LA  4  5   6   7\n",
        "SF  8  9  10  11"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Just like a Series, axis indexes can also use map\n",
      "\n",
      "#Let's use map to lowercase the city initials\n",
      "dframe.index.map(str.lower)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 36,
       "text": [
        "array(['ny', 'la', 'sf'], dtype=object)"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# If you want to assign this to the actual index, you can use index\n",
      "dframe.index = dframe.index.map(str.lower)\n",
      "#Show\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>ny</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>la</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sf</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 38,
       "text": [
        "    A  B   C   D\n",
        "ny  0  1   2   3\n",
        "la  4  5   6   7\n",
        "sf  8  9  10  11"
       ]
      }
     ],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Use rename if you want to create a transformed version withour modifying the original!\n",
      "\n",
      "#str.title will capitalize the first letter, lowercasing the columns\n",
      "dframe.rename(index=str.title, columns=str.lower)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>a</th>\n",
        "      <th>b</th>\n",
        "      <th>c</th>\n",
        "      <th>d</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>Ny</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>La</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>Sf</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 39,
       "text": [
        "    a  b   c   d\n",
        "Ny  0  1   2   3\n",
        "La  4  5   6   7\n",
        "Sf  8  9  10  11"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also use rename to insert dictionaries providing new values for indexes or columns!\n",
      "dframe.rename(index={'ny': 'NEW YORK'},\n",
      "            columns={'A': 'ALPHA'})"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>ALPHA</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>NEW YORK</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>la</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sf</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 40,
       "text": [
        "          ALPHA  B   C   D\n",
        "NEW YORK      0  1   2   3\n",
        "la            4  5   6   7\n",
        "sf            8  9  10  11"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# If you would like to actually edit the data set in place, set inplace=True\n",
      "dframe.rename(index={'ny': 'NEW YORK'}, inplace=True)\n",
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>A</th>\n",
        "      <th>B</th>\n",
        "      <th>C</th>\n",
        "      <th>D</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>NEW YORK</th>\n",
        "      <td> 0</td>\n",
        "      <td> 1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>la</th>\n",
        "      <td> 4</td>\n",
        "      <td> 5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sf</th>\n",
        "      <td> 8</td>\n",
        "      <td> 9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 41,
       "text": [
        "          A  B   C   D\n",
        "NEW YORK  0  1   2   3\n",
        "la        4  5   6   7\n",
        "sf        8  9  10  11"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Binning"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "years = [1990,1991,1992,2008,2012,2015,1987,1969,2013,2008,1999]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can seperate these years by decade\n",
      "decade_bins = [1960,1970,1980,1990,2000,2010,2020]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 43
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now we'll use cut to get somethign called a Category object\n",
      "decade_cat = pd.cut(years,decade_bins)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "decade_cat"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 45,
       "text": [
        " (1980, 1990]\n",
        " (1990, 2000]\n",
        " (1990, 2000]\n",
        " (2000, 2010]\n",
        " (2010, 2020]\n",
        " (2010, 2020]\n",
        " (1980, 1990]\n",
        " (1960, 1970]\n",
        " (2010, 2020]\n",
        " (2000, 2010]\n",
        " (1990, 2000]\n",
        "Levels (6): Index(['(1960, 1970]', '(1970, 1980]', '(1980, 1990]',\n",
        "                   '(1990, 2000]', '(2000, 2010]', '(2010, 2020]'], dtype=object)"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Then we can check the value counts in each category\n",
      "pd.value_counts(decade_cat)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 48,
       "text": [
        "(2010, 2020]    3\n",
        "(1990, 2000]    3\n",
        "(2000, 2010]    2\n",
        "(1980, 1990]    2\n",
        "(1960, 1970]    1\n",
        "dtype: int64"
       ]
      }
     ],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# We can also pass data values to the cut.\n",
      "\n",
      "#For instance, if we just wanted to make two bins, evenly spaced based on max and min year, with a 1 year precision\n",
      "pd.cut(years,2,precision=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 49,
       "text": [
        " (1969, 1992]\n",
        " (1969, 1992]\n",
        " (1969, 1992]\n",
        " (1992, 2015]\n",
        " (1992, 2015]\n",
        " (1992, 2015]\n",
        " (1969, 1992]\n",
        " (1969, 1992]\n",
        " (1992, 2015]\n",
        " (1992, 2015]\n",
        " (1992, 2015]\n",
        "Levels (2): Index(['(1969, 1992]', '(1992, 2015]'], dtype=object)"
       ]
      }
     ],
     "prompt_number": 49
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Finding Outliers and Describing Data"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let's see how we would find outliers in a dataset\n",
      "# random generator\n",
      "# First we'll seed the numpy generator\n",
      "np.random.seed(12345)\n",
      "\n",
      "#Next we'll create the dataframe\n",
      "dframe = DataFrame(np.random.randn(1000,4))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 50
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Show preview\n",
      "dframe.head()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>-0.204708</td>\n",
        "      <td> 0.478943</td>\n",
        "      <td>-0.519439</td>\n",
        "      <td>-0.555730</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td> 1.965781</td>\n",
        "      <td> 1.393406</td>\n",
        "      <td> 0.092908</td>\n",
        "      <td> 0.281746</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td> 0.769023</td>\n",
        "      <td> 1.246435</td>\n",
        "      <td> 1.007189</td>\n",
        "      <td>-1.296221</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 0.274992</td>\n",
        "      <td> 0.228913</td>\n",
        "      <td> 1.352917</td>\n",
        "      <td> 0.886429</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
        "      <td>-2.001637</td>\n",
        "      <td>-0.371843</td>\n",
        "      <td> 1.669025</td>\n",
        "      <td>-0.438570</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 51,
       "text": [
        "          0         1         2         3\n",
        "0 -0.204708  0.478943 -0.519439 -0.555730\n",
        "1  1.965781  1.393406  0.092908  0.281746\n",
        "2  0.769023  1.246435  1.007189 -1.296221\n",
        "3  0.274992  0.228913  1.352917  0.886429\n",
        "4 -2.001637 -0.371843  1.669025 -0.438570"
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets describe the data\n",
      "dframe.describe()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>count</th>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>mean</th>\n",
        "      <td>   -0.067684</td>\n",
        "      <td>    0.067924</td>\n",
        "      <td>    0.025598</td>\n",
        "      <td>   -0.002298</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>std</th>\n",
        "      <td>    0.998035</td>\n",
        "      <td>    0.992106</td>\n",
        "      <td>    1.006835</td>\n",
        "      <td>    0.996794</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>min</th>\n",
        "      <td>   -3.428254</td>\n",
        "      <td>   -3.548824</td>\n",
        "      <td>   -3.184377</td>\n",
        "      <td>   -3.745356</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25%</th>\n",
        "      <td>   -0.774890</td>\n",
        "      <td>   -0.591841</td>\n",
        "      <td>   -0.641675</td>\n",
        "      <td>   -0.644144</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>50%</th>\n",
        "      <td>   -0.116401</td>\n",
        "      <td>    0.101143</td>\n",
        "      <td>    0.002073</td>\n",
        "      <td>   -0.013611</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>75%</th>\n",
        "      <td>    0.616366</td>\n",
        "      <td>    0.780282</td>\n",
        "      <td>    0.680391</td>\n",
        "      <td>    0.654328</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>max</th>\n",
        "      <td>    3.366626</td>\n",
        "      <td>    2.653656</td>\n",
        "      <td>    3.260383</td>\n",
        "      <td>    3.927528</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 52,
       "text": [
        "                 0            1            2            3\n",
        "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
        "mean     -0.067684     0.067924     0.025598    -0.002298\n",
        "std       0.998035     0.992106     1.006835     0.996794\n",
        "min      -3.428254    -3.548824    -3.184377    -3.745356\n",
        "25%      -0.774890    -0.591841    -0.641675    -0.644144\n",
        "50%      -0.116401     0.101143     0.002073    -0.013611\n",
        "75%       0.616366     0.780282     0.680391     0.654328\n",
        "max       3.366626     2.653656     3.260383     3.927528"
       ]
      }
     ],
     "prompt_number": 52
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Lets select the first column\n",
      "col = dframe[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 53
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# NOw we can check which values in the column are greater than 3, for instance.\n",
      "col[np.abs(col)>3]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 54,
       "text": [
        "523   -3.428254\n",
        "900    3.366626\n",
        "Name: 0, dtype: float64"
       ]
      }
     ],
     "prompt_number": 54
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# So we now know in column[0], rows 523 and 900 have values with abs > 3\n",
      "\n",
      "#How about all the columns?\n",
      "\n",
      "# We can use the \"any\" method\n",
      "dframe[(np.abs(dframe)>3).any(1)]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>5  </th>\n",
        "      <td>-0.539741</td>\n",
        "      <td> 0.476985</td>\n",
        "      <td> 3.248944</td>\n",
        "      <td>-1.021228</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>97 </th>\n",
        "      <td>-0.774363</td>\n",
        "      <td> 0.552936</td>\n",
        "      <td> 0.106061</td>\n",
        "      <td> 3.927528</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>102</th>\n",
        "      <td>-0.655054</td>\n",
        "      <td>-0.565230</td>\n",
        "      <td> 3.176873</td>\n",
        "      <td> 0.959533</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>305</th>\n",
        "      <td>-2.315555</td>\n",
        "      <td> 0.457246</td>\n",
        "      <td>-0.025907</td>\n",
        "      <td>-3.399312</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>324</th>\n",
        "      <td> 0.050188</td>\n",
        "      <td> 1.951312</td>\n",
        "      <td> 3.260383</td>\n",
        "      <td> 0.963301</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>400</th>\n",
        "      <td> 0.146326</td>\n",
        "      <td> 0.508391</td>\n",
        "      <td>-0.196713</td>\n",
        "      <td>-3.745356</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>499</th>\n",
        "      <td>-0.293333</td>\n",
        "      <td>-0.242459</td>\n",
        "      <td>-3.056990</td>\n",
        "      <td> 1.918403</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>523</th>\n",
        "      <td>-3.428254</td>\n",
        "      <td>-0.296336</td>\n",
        "      <td>-0.439938</td>\n",
        "      <td>-0.867165</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>586</th>\n",
        "      <td> 0.275144</td>\n",
        "      <td> 1.179227</td>\n",
        "      <td>-3.184377</td>\n",
        "      <td> 1.369891</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>808</th>\n",
        "      <td>-0.362528</td>\n",
        "      <td>-3.548824</td>\n",
        "      <td> 1.553205</td>\n",
        "      <td>-2.186301</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>900</th>\n",
        "      <td> 3.366626</td>\n",
        "      <td>-2.372214</td>\n",
        "      <td> 0.851010</td>\n",
        "      <td> 1.332846</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 55,
       "text": [
        "            0         1         2         3\n",
        "5   -0.539741  0.476985  3.248944 -1.021228\n",
        "97  -0.774363  0.552936  0.106061  3.927528\n",
        "102 -0.655054 -0.565230  3.176873  0.959533\n",
        "305 -2.315555  0.457246 -0.025907 -3.399312\n",
        "324  0.050188  1.951312  3.260383  0.963301\n",
        "400  0.146326  0.508391 -0.196713 -3.745356\n",
        "499 -0.293333 -0.242459 -3.056990  1.918403\n",
        "523 -3.428254 -0.296336 -0.439938 -0.867165\n",
        "586  0.275144  1.179227 -3.184377  1.369891\n",
        "808 -0.362528 -3.548824  1.553205 -2.186301\n",
        "900  3.366626 -2.372214  0.851010  1.332846"
       ]
      }
     ],
     "prompt_number": 55
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# WE could also possibly cap the data at 3\n",
      "\n",
      "dframe[np.abs(dframe)>3] = np.sign(dframe) *3"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 56
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe.describe()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>count</th>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "      <td> 1000.000000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>mean</th>\n",
        "      <td>   -0.067623</td>\n",
        "      <td>    0.068473</td>\n",
        "      <td>    0.025153</td>\n",
        "      <td>   -0.002081</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>std</th>\n",
        "      <td>    0.995485</td>\n",
        "      <td>    0.990253</td>\n",
        "      <td>    1.003977</td>\n",
        "      <td>    0.989736</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>min</th>\n",
        "      <td>   -3.000000</td>\n",
        "      <td>   -3.000000</td>\n",
        "      <td>   -3.000000</td>\n",
        "      <td>   -3.000000</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>25%</th>\n",
        "      <td>   -0.774890</td>\n",
        "      <td>   -0.591841</td>\n",
        "      <td>   -0.641675</td>\n",
        "      <td>   -0.644144</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>50%</th>\n",
        "      <td>   -0.116401</td>\n",
        "      <td>    0.101143</td>\n",
        "      <td>    0.002073</td>\n",
        "      <td>   -0.013611</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>75%</th>\n",
        "      <td>    0.616366</td>\n",
        "      <td>    0.780282</td>\n",
        "      <td>    0.680391</td>\n",
        "      <td>    0.654328</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>max</th>\n",
        "      <td>    3.000000</td>\n",
        "      <td>    2.653656</td>\n",
        "      <td>    3.000000</td>\n",
        "      <td>    3.000000</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 57,
       "text": [
        "                 0            1            2            3\n",
        "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
        "mean     -0.067623     0.068473     0.025153    -0.002081\n",
        "std       0.995485     0.990253     1.003977     0.989736\n",
        "min      -3.000000    -3.000000    -3.000000    -3.000000\n",
        "25%      -0.774890    -0.591841    -0.641675    -0.644144\n",
        "50%      -0.116401     0.101143     0.002073    -0.013611\n",
        "75%       0.616366     0.780282     0.680391     0.654328\n",
        "max       3.000000     2.653656     3.000000     3.000000"
       ]
      }
     ],
     "prompt_number": 57
    },
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Permutation"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# WE can randomly reorder (permutate) a Series, or the rows in a DataFrame\n",
      "\n",
      "#Let's take a look\n",
      "dframe = DataFrame(np.arange(4 * 4).reshape((4, 4)))\n",
      "\n",
      "#Create an array with a random perumation of 0,1,2,3\n",
      "blender = np.random.permutation(4)\n",
      "\n",
      "blender"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 58,
       "text": [
        "array([1, 3, 2, 0])"
       ]
      }
     ],
     "prompt_number": 58
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dframe"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  4</td>\n",
        "      <td>  5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 60,
       "text": [
        "    0   1   2   3\n",
        "0   0   1   2   3\n",
        "1   4   5   6   7\n",
        "2   8   9  10  11\n",
        "3  12  13  14  15"
       ]
      }
     ],
     "prompt_number": 60
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Now permutate the dframe based on the blender\n",
      "#permutation of the rows\n",
      "dframe.take(blender)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>0</th>\n",
        "      <th>1</th>\n",
        "      <th>2</th>\n",
        "      <th>3</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>  4</td>\n",
        "      <td>  5</td>\n",
        "      <td>  6</td>\n",
        "      <td>  7</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>3</th>\n",
        "      <td> 12</td>\n",
        "      <td> 13</td>\n",
        "      <td> 14</td>\n",
        "      <td> 15</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>  8</td>\n",
        "      <td>  9</td>\n",
        "      <td> 10</td>\n",
        "      <td> 11</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>  0</td>\n",
        "      <td>  1</td>\n",
        "      <td>  2</td>\n",
        "      <td>  3</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 61,
       "text": [
        "    0   1   2   3\n",
        "1   4   5   6   7\n",
        "3  12  13  14  15\n",
        "2   8   9  10  11\n",
        "0   0   1   2   3"
       ]
      }
     ],
     "prompt_number": 61
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Let imagine a box with 3 marbles in it: labeled 1, 2, and 3\n",
      "box = np.array([1,2,3])\n",
      "\n",
      "# Now lets create a random permuation WITH replacement using randint\n",
      "shaker = np.random.randint(0, len(box), size=10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 62
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "shaker"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 63,
       "text": [
        "array([2, 0, 2, 0, 2, 0, 2, 0, 2, 2])"
       ]
      }
     ],
     "prompt_number": 63
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "box"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 64,
       "text": [
        "array([1, 2, 3])"
       ]
      }
     ],
     "prompt_number": 64
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Now lets grab form the box\n",
      "hand_grabs = box.take(shaker)\n",
      "\n",
      "#show\n",
      "hand_grabs"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 65,
       "text": [
        "array([3, 1, 3, 1, 3, 1, 3, 1, 3, 3])"
       ]
      }
     ],
     "prompt_number": 65
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}