Guided Project - Winning Jeopardy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Show Number    Air Date      Round                         Category  Value  \\\n",
       "0         4680  2004-12-31  Jeopardy!                          HISTORY   $200   \n",
       "1         4680  2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES   $200   \n",
       "2         4680  2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...   $200   \n",
       "3         4680  2004-12-31  Jeopardy!                 THE COMPANY LINE   $200   \n",
       "4         4680  2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES   $200   \n",
       "\n",
       "                                            Question      Answer  \n",
       "0  For the last 8 years of his life, Galileo was ...  Copernicus  \n",
       "1  No. 2: 1912 Olympian; football star at Carlisl...  Jim Thorpe  \n",
       "2  The city of Yuma in this state has a record av...     Arizona  \n",
       "3  In 1963, live on \"The Art Linkletter Show\", th...  McDonald's  \n",
       "4  Signer of the Dec. of Indep., framer of the Co...  John Adams  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy  = pd.read_csv('jeopardy.csv')\n",
    "jeopardy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Show Number', ' Air Date', ' Round', ' Category', ' Value',\n",
       "       ' Question', ' Answer'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Show Number', 'Air Date', 'Round', 'Category', 'Value', 'Question',\n",
       "       'Answer'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.columns = ['Show Number', 'Air Date', 'Round', 'Category', 'Value', 'Question', 'Answer']\n",
    "jeopardy.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def normalize_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(\"[^A-Za-z0-9\\s]\", \"\", text)\n",
    "    return text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jeopardy[\"clean_question\"] = jeopardy[\"Question\"].apply(normalize_text)\n",
    "jeopardy[\"clean_answer\"] = jeopardy[\"Answer\"].apply(normalize_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def normalize_values(text):\n",
    "    text = re.sub(\"[^A-Za-z0-9\\s]\", \"\", text)\n",
    "    try:\n",
    "        text = int(text)\n",
    "    except Exception:\n",
    "        text = 0\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jeopardy[\"clean_value\"] = jeopardy[\"Value\"].apply(normalize_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>clean_question</th>\n",
       "      <th>clean_answer</th>\n",
       "      <th>clean_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "      <td>for the last 8 years of his life galileo was u...</td>\n",
       "      <td>copernicus</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "      <td>no 2 1912 olympian football star at carlisle i...</td>\n",
       "      <td>jim thorpe</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>the city of yuma in this state has a record av...</td>\n",
       "      <td>arizona</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "      <td>in 1963 live on the art linkletter show this c...</td>\n",
       "      <td>mcdonalds</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "      <td>signer of the dec of indep framer of the const...</td>\n",
       "      <td>john adams</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Show Number    Air Date      Round                         Category Value  \\\n",
       "0         4680  2004-12-31  Jeopardy!                          HISTORY  $200   \n",
       "1         4680  2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES  $200   \n",
       "2         4680  2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...  $200   \n",
       "3         4680  2004-12-31  Jeopardy!                 THE COMPANY LINE  $200   \n",
       "4         4680  2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES  $200   \n",
       "\n",
       "                                            Question      Answer  \\\n",
       "0  For the last 8 years of his life, Galileo was ...  Copernicus   \n",
       "1  No. 2: 1912 Olympian; football star at Carlisl...  Jim Thorpe   \n",
       "2  The city of Yuma in this state has a record av...     Arizona   \n",
       "3  In 1963, live on \"The Art Linkletter Show\", th...  McDonald's   \n",
       "4  Signer of the Dec. of Indep., framer of the Co...  John Adams   \n",
       "\n",
       "                                      clean_question clean_answer  clean_value  \n",
       "0  for the last 8 years of his life galileo was u...   copernicus          200  \n",
       "1  no 2 1912 olympian football star at carlisle i...   jim thorpe          200  \n",
       "2  the city of yuma in this state has a record av...      arizona          200  \n",
       "3  in 1963 live on the art linkletter show this c...    mcdonalds          200  \n",
       "4  signer of the dec of indep framer of the const...   john adams          200  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "jeopardy[\"Air Date\"] = pd.to_datetime(jeopardy[\"Air Date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def match_counts(row_string):\n",
    "    split_answer = row_string['clean_answer'].split(\" \")\n",
    "    split_question = row_string['clean_question'].split(\" \")\n",
    "    match_count = 0\n",
    "    if \"the\" in split_answer:\n",
    "        split_answer.remove(\"the\")\n",
    "    if len(split_answer) == 0:\n",
    "        return\n",
    "    for split_answer_split in split_answer:\n",
    "        if split_answer_split in split_question:\n",
    "            match_count += 1\n",
    "    return match_count / len(split_answer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "jeopardy[\"answer_in_question\"] = jeopardy.apply(match_counts, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06049628203468587"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy[\"answer_in_question\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "question_overlap = []\n",
    "terms_used = set()\n",
    "for i, row in jeopardy.iterrows():\n",
    "        split_question = row[\"clean_question\"].split(\" \")\n",
    "        split_question = [q for q in split_question if len(q) > 5]\n",
    "        match_count = 0\n",
    "        for word in split_question:\n",
    "            if word in terms_used:\n",
    "                match_count += 1\n",
    "        for word in split_question:\n",
    "            terms_used.add(word)\n",
    "        if len(split_question) > 0:\n",
    "            match_count /= len(split_question)\n",
    "        question_overlap.append(match_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jeopardy[\"question_overlap\"] = question_overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6908737315671962"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy[\"question_overlap\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def determine_value(row):\n",
    "    value = 0\n",
    "    if row[\"clean_value\"] > 800:\n",
    "        value = 1\n",
    "    return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jeopardy[\"high_value\"] = jeopardy.apply(determine_value, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Show Number</th>\n",
       "      <th>Air Date</th>\n",
       "      <th>Round</th>\n",
       "      <th>Category</th>\n",
       "      <th>Value</th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "      <th>clean_question</th>\n",
       "      <th>clean_answer</th>\n",
       "      <th>clean_value</th>\n",
       "      <th>answer_in_question</th>\n",
       "      <th>question_overlap</th>\n",
       "      <th>high_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>HISTORY</td>\n",
       "      <td>$200</td>\n",
       "      <td>For the last 8 years of his life, Galileo was ...</td>\n",
       "      <td>Copernicus</td>\n",
       "      <td>for the last 8 years of his life galileo was u...</td>\n",
       "      <td>copernicus</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>ESPN's TOP 10 ALL-TIME ATHLETES</td>\n",
       "      <td>$200</td>\n",
       "      <td>No. 2: 1912 Olympian; football star at Carlisl...</td>\n",
       "      <td>Jim Thorpe</td>\n",
       "      <td>no 2 1912 olympian football star at carlisle i...</td>\n",
       "      <td>jim thorpe</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EVERYBODY TALKS ABOUT IT...</td>\n",
       "      <td>$200</td>\n",
       "      <td>The city of Yuma in this state has a record av...</td>\n",
       "      <td>Arizona</td>\n",
       "      <td>the city of yuma in this state has a record av...</td>\n",
       "      <td>arizona</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>THE COMPANY LINE</td>\n",
       "      <td>$200</td>\n",
       "      <td>In 1963, live on \"The Art Linkletter Show\", th...</td>\n",
       "      <td>McDonald's</td>\n",
       "      <td>in 1963 live on the art linkletter show this c...</td>\n",
       "      <td>mcdonalds</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4680</td>\n",
       "      <td>2004-12-31</td>\n",
       "      <td>Jeopardy!</td>\n",
       "      <td>EPITAPHS &amp; TRIBUTES</td>\n",
       "      <td>$200</td>\n",
       "      <td>Signer of the Dec. of Indep., framer of the Co...</td>\n",
       "      <td>John Adams</td>\n",
       "      <td>signer of the dec of indep framer of the const...</td>\n",
       "      <td>john adams</td>\n",
       "      <td>200</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Show Number   Air Date      Round                         Category Value  \\\n",
       "0         4680 2004-12-31  Jeopardy!                          HISTORY  $200   \n",
       "1         4680 2004-12-31  Jeopardy!  ESPN's TOP 10 ALL-TIME ATHLETES  $200   \n",
       "2         4680 2004-12-31  Jeopardy!      EVERYBODY TALKS ABOUT IT...  $200   \n",
       "3         4680 2004-12-31  Jeopardy!                 THE COMPANY LINE  $200   \n",
       "4         4680 2004-12-31  Jeopardy!              EPITAPHS & TRIBUTES  $200   \n",
       "\n",
       "                                            Question      Answer  \\\n",
       "0  For the last 8 years of his life, Galileo was ...  Copernicus   \n",
       "1  No. 2: 1912 Olympian; football star at Carlisl...  Jim Thorpe   \n",
       "2  The city of Yuma in this state has a record av...     Arizona   \n",
       "3  In 1963, live on \"The Art Linkletter Show\", th...  McDonald's   \n",
       "4  Signer of the Dec. of Indep., framer of the Co...  John Adams   \n",
       "\n",
       "                                      clean_question clean_answer  \\\n",
       "0  for the last 8 years of his life galileo was u...   copernicus   \n",
       "1  no 2 1912 olympian football star at carlisle i...   jim thorpe   \n",
       "2  the city of yuma in this state has a record av...      arizona   \n",
       "3  in 1963 live on the art linkletter show this c...    mcdonalds   \n",
       "4  signer of the dec of indep framer of the const...   john adams   \n",
       "\n",
       "   clean_value  answer_in_question  question_overlap  high_value  \n",
       "0          200                 0.0               0.0           0  \n",
       "1          200                 0.0               0.0           0  \n",
       "2          200                 0.0               0.0           0  \n",
       "3          200                 0.0               0.0           0  \n",
       "4          200                 0.0               0.0           0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jeopardy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def count_usage(term):\n",
    "    low_count = 0\n",
    "    high_count = 0\n",
    "    for i, row in jeopardy.iterrows():\n",
    "        if term in row[\"clean_question\"].split(\" \"):\n",
    "            if row[\"high_value\"] == 1:\n",
    "                high_count += 1\n",
    "            else:\n",
    "                low_count += 1\n",
    "    return high_count, low_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 0), (0, 1), (1, 1), (2, 0), (0, 1)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comparison_terms = list(terms_used)[:5]\n",
    "observed_expected = []\n",
    "for term in comparison_terms:\n",
    "    observed_expected.append(count_usage(term))\n",
    "\n",
    "observed_expected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "high_value_count = jeopardy[jeopardy[\"high_value\"] == 1].shape[0]\n",
    "low_value_count = jeopardy[jeopardy[\"high_value\"] == 0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14265"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "high_value_count\n",
    "low_value_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from scipy.stats import chisquare\n",
    "import numpy as np\n",
    "chi_squared = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for obs in observed_expected:\n",
    "    total = sum(obs)\n",
    "    total_prop = total / jeopardy.shape[0]\n",
    "    high_value_exp = total_prop * high_value_count\n",
    "    low_value_exp = total_prop * low_value_count\n",
    "    \n",
    "    observed = np.array([obs[0], obs[1]])\n",
    "    expected = np.array([high_value_exp, low_value_exp])\n",
    "    chi_squared.append(chisquare(observed, expected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Power_divergenceResult(statistic=2.487792117195675, pvalue=0.11473257634454047),\n",
       " Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469),\n",
       " Power_divergenceResult(statistic=0.4448774816612795, pvalue=0.5047776487545996),\n",
       " Power_divergenceResult(statistic=4.97558423439135, pvalue=0.025707519787911092),\n",
       " Power_divergenceResult(statistic=0.401962846126884, pvalue=0.5260772985705469)]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chi_squared"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}