Notebook 2 - Exploratory Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import the necessary packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the path to where the munged data is located."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path_to_data = '/home/bsknight/Documents/Personal_Training_Git/Udacity/Udacity-Machine_Learning_Nanodegree/' + \\\n",
    "               'Capstone_Project/Data' \n",
    "os.chdir(path_to_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('./munged_df.csv', index_col='account_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>/blog/should-you-outsource-your-order-fulfillment-processpage_view</th>\n",
       "      <th>about/page_view</th>\n",
       "      <th>blog/allpage_view</th>\n",
       "      <th>blog/black-fridays-nowpage_view</th>\n",
       "      <th>blog/topic/customer-spotlightpage_view</th>\n",
       "      <th>blog/topic/inventory-managementpage_view</th>\n",
       "      <th>blog/topic/inventory-setupspage_view</th>\n",
       "      <th>blog/topic/master-your-datapage_view</th>\n",
       "      <th>blog/topic/multichannel-operationspage_view</th>\n",
       "      <th>blog/topic/stitch-updatespage_view</th>\n",
       "      <th>...</th>\n",
       "      <th>mkt_source_woocommerce</th>\n",
       "      <th>mkt_source_woocommerce_com</th>\n",
       "      <th>mkt_source_woothemes_com</th>\n",
       "      <th>mkt_source_xero</th>\n",
       "      <th>mkt_source_xero_com</th>\n",
       "      <th>mkt_source_youtube_com</th>\n",
       "      <th>mobile_pageviews</th>\n",
       "      <th>non_mobile_pageviews</th>\n",
       "      <th>share_of_pageviews</th>\n",
       "      <th>cc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>account_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1206.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1276.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>138.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1286.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1544.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1628.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 582 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            /blog/should-you-outsource-your-order-fulfillment-processpage_view  \\\n",
       "account_id                                                                       \n",
       "1206.0                                                      0                    \n",
       "1276.0                                                      0                    \n",
       "1286.0                                                      0                    \n",
       "1544.0                                                      0                    \n",
       "1628.0                                                      0                    \n",
       "\n",
       "            about/page_view  blog/allpage_view  \\\n",
       "account_id                                       \n",
       "1206.0                    0                  0   \n",
       "1276.0                    0                  0   \n",
       "1286.0                    0                  0   \n",
       "1544.0                    0                  0   \n",
       "1628.0                    0                  0   \n",
       "\n",
       "            blog/black-fridays-nowpage_view  \\\n",
       "account_id                                    \n",
       "1206.0                                    0   \n",
       "1276.0                                    0   \n",
       "1286.0                                    0   \n",
       "1544.0                                    0   \n",
       "1628.0                                    0   \n",
       "\n",
       "            blog/topic/customer-spotlightpage_view  \\\n",
       "account_id                                           \n",
       "1206.0                                           0   \n",
       "1276.0                                           0   \n",
       "1286.0                                           0   \n",
       "1544.0                                           0   \n",
       "1628.0                                           0   \n",
       "\n",
       "            blog/topic/inventory-managementpage_view  \\\n",
       "account_id                                             \n",
       "1206.0                                             0   \n",
       "1276.0                                             0   \n",
       "1286.0                                             0   \n",
       "1544.0                                             0   \n",
       "1628.0                                             0   \n",
       "\n",
       "            blog/topic/inventory-setupspage_view  \\\n",
       "account_id                                         \n",
       "1206.0                                         0   \n",
       "1276.0                                         0   \n",
       "1286.0                                         0   \n",
       "1544.0                                         0   \n",
       "1628.0                                         0   \n",
       "\n",
       "            blog/topic/master-your-datapage_view  \\\n",
       "account_id                                         \n",
       "1206.0                                         0   \n",
       "1276.0                                         0   \n",
       "1286.0                                         0   \n",
       "1544.0                                         0   \n",
       "1628.0                                         0   \n",
       "\n",
       "            blog/topic/multichannel-operationspage_view  \\\n",
       "account_id                                                \n",
       "1206.0                                                0   \n",
       "1276.0                                                0   \n",
       "1286.0                                                0   \n",
       "1544.0                                                0   \n",
       "1628.0                                                0   \n",
       "\n",
       "            blog/topic/stitch-updatespage_view ...   mkt_source_woocommerce  \\\n",
       "account_id                                     ...                            \n",
       "1206.0                                       0 ...                      0.0   \n",
       "1276.0                                       0 ...                      0.0   \n",
       "1286.0                                       0 ...                      0.0   \n",
       "1544.0                                       0 ...                      0.0   \n",
       "1628.0                                       0 ...                      0.0   \n",
       "\n",
       "            mkt_source_woocommerce_com  mkt_source_woothemes_com  \\\n",
       "account_id                                                         \n",
       "1206.0                             0.0                       0.0   \n",
       "1276.0                             0.0                       0.0   \n",
       "1286.0                             0.0                       0.0   \n",
       "1544.0                             0.0                       0.0   \n",
       "1628.0                             0.0                       0.0   \n",
       "\n",
       "            mkt_source_xero  mkt_source_xero_com  mkt_source_youtube_com  \\\n",
       "account_id                                                                 \n",
       "1206.0                  0.0                  0.0                     0.0   \n",
       "1276.0                  0.0                  0.0                     0.0   \n",
       "1286.0                  0.0                  0.0                     0.0   \n",
       "1544.0                  0.0                  0.0                     0.0   \n",
       "1628.0                  0.0                  0.0                     0.0   \n",
       "\n",
       "            mobile_pageviews  non_mobile_pageviews  share_of_pageviews   cc  \n",
       "account_id                                                                   \n",
       "1206.0                   0.0                 215.0                 0.0  0.0  \n",
       "1276.0                   0.0                 138.0                 0.0  0.0  \n",
       "1286.0                   0.0                  64.0                 0.0  0.0  \n",
       "1544.0                   0.0                   1.0                 0.0  0.0  \n",
       "1628.0                   0.0                  15.0                 0.0  0.0  \n",
       "\n",
       "[5 rows x 582 columns]"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the size of the data set?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Each observation is an indivual account\n",
      "Our data set has 16607 observations.\n",
      "Our data set has 581 features.\n",
      "The probability of a label designating a sucessful conversion to paying customer is 0.0602155717469.\n"
     ]
    }
   ],
   "source": [
    "print 'Each observation is an indivual account'\n",
    "print 'Our data set has ' + str(len(df)) + ' observations.'\n",
    "print 'Our data set has ' + str(len(list(df)) - 1) + ' features.'\n",
    "print 'The probability of a label designating a sucessful conversion to paying customer is ' + \\\n",
    "     str(df['cc'].sum() / len(df)) + '.'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The labels for our data are contained in the 'cc' column. Let's see how frequent the labels are."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a horizontal bar chart."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(8,3))\n",
    "a = pd.DataFrame(pd.Series(df[\"cc\"]).value_counts()).iloc[0]['cc']\n",
    "b = pd.DataFrame(pd.Series(df[\"cc\"]).value_counts()).iloc[1]['cc']\n",
    "plt.barh([0], [a], label=\"No Credit Card Added\", color='grey', align='center', alpha=0.5)\n",
    "plt.barh([1], [b], label=\"Credit Card Added\", color = 'green', align='center', alpha=0.5)\n",
    "y_labels = [\"No Credit Card Added\", \"Credit Card Added\"]\n",
    "x_values = np.arange(0,16001,1000)\n",
    "x_labels= ['0', '1,000', '2,000', '3,000', '4,000', '5,000', '6,000', '7,000', '8,000', '9,000', '10,000', \n",
    "           '11,000', '12,000', '13,000', '14,000', '15,000', '16,000']\n",
    "plt.xticks(x_values, x_labels, rotation=45)\n",
    "plt.yticks([0,1], y_labels)\n",
    "plt.ylabel('Account Status', rotation='horizontal', fontsize=13)\n",
    "plt.xlabel('# of Accounts Created (November 2015 to January 2017)', fontsize=13)\n",
    "# plt.title('The Observation Labels: Conversion to Paying Account', fontsize=18, y=1.01, x=0.3)\n",
    "plt.tight_layout()\n",
    "# plt.legend()\n",
    "# plt.show()\n",
    "path = '/home/bsknight/Documents/Personal_Training_Git/Udacity/Udacity-Machine_Learning_Nanodegree/' + \\\n",
    "       'Capstone_Project/Images'\n",
    "savepath = os.path.join(path, 'exploratory_analysis-labels.png')\n",
    "plt.savefig(savepath)\n",
    "plt.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a pie chart."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(16,7.5))\n",
    "# plt.rcParams['font.size'] = 24.0\n",
    "# labels = 'Converted to \\n Paying Customer \\n (Added a Credit Card)', 'Never Converted  \\n (No Credit Card Added)'\n",
    "# sizes = [df[\"cc\"].sum(), (len(df)-df[\"cc\"].sum()) ]\n",
    "# colors = ['g', '#7F7F7F']\n",
    "# explode = (0.0, 0.4)  # explode 1st slice\n",
    "\n",
    "# plt.pie(sizes, explode=explode, labels=labels, colors=colors,\n",
    "#         autopct='%1.1f%%', shadow=True, startangle=140)\n",
    "# # plt.title('City:[3,500, 6,800 Students] vs. Metro:[6,800, 712,131 Students]', \n",
    "# #           y=1, fontsize=20)\n",
    "# # plt.suptitle('District Segementation', y=0.98, fontsize=25)\n",
    "# # plt.show()\n",
    "\n",
    "# plt.axis('equal')\n",
    "# # plt.show()\n",
    "# path = '/home/bsknight/Documents/Personal_Training_Git/Udacity/Udacity-Machine_Learning_Nanodegree/' + \\\n",
    "#        'Capstone_Project/Images'\n",
    "# savepath = os.path.join(path, 'exploratory_analysis-labels.png')\n",
    "# plt.savefig(savepath)\n",
    "# plt.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we've examined the distrubution of labels, lets calculate the summary statistics for the other features. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "means = pd.DataFrame(df.mean())\n",
    "means = means.reset_index()\n",
    "means = means.rename(columns={'index': 'Feature', 0: 'Mean'})\n",
    "means.sort_values(by='Mean', ascending=0, inplace=True)\n",
    "bins = np.arange(0, 30, 1)\n",
    "plt.hist(means[\"Mean\"], bins, histtype='bar', alpha=0.5)\n",
    "plt.xlabel('Feature Mean')\n",
    "plt.ylabel('Number of Features')\n",
    "plt.title('Histogram of Feature Means (Bin = 1)')\n",
    "plt.tight_layout()\n",
    "# plt.ylim((0,30))\n",
    "# plt.show()\n",
    "savepath = os.path.join(path, 'exploratory_analysis-feature_means.png')\n",
    "plt.savefig(savepath)\n",
    "plt.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now for the standard deviations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sds = pd.DataFrame(df.std())  \n",
    "sds = sds.reset_index()\n",
    "sds = sds.rename(columns={'index': 'Feature', 0: 'SD'})\n",
    "sds.sort_values(by='SD', ascending=0, inplace=True)\n",
    "bins = np.arange(0, 200, 10)\n",
    "plt.hist(sds[\"SD\"], bins, histtype='bar', alpha=0.5)\n",
    "plt.xlabel('Feature Standard Deviation')\n",
    "plt.ylabel('Number of Features')\n",
    "plt.title('Histogram of Feature Standard Deviations (Bin = 10)')\n",
    "plt.tight_layout()\n",
    "# plt.ylim((0,100))\n",
    "# plt.show()\n",
    "savepath = os.path.join(path, 'exploratory_analysis-feature_sds.png')\n",
    "plt.savefig(savepath)\n",
    "plt.clf()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py2]",
   "language": "python",
   "name": "conda-env-py2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}