Single State Scatter Text - TEXAS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#custom functions \n",
    "from projectfunctions import * "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd  \n",
    "import spacy \n",
    "import scattertext as st \n",
    "from IPython.core.display import display, HTML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in the NGSS and Arizona state standards & apply preprocessing methods including tokenizing, removing stop words to the text. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ngss corpus\n",
    "ngss = open_and_flatten('TXTfiles/ngss')   \n",
    "\n",
    "#create a nested list\n",
    "s = ['ngss'] \n",
    "s.append(ngss) \n",
    "\n",
    "#create a ngss dataframe\n",
    "ngss_corpi = pd.DataFrame(s, columns=[\"corpus\"]) \n",
    "ngss_corpi['state'] = \"ngss\" \n",
    "ngss_corpi = ngss_corpi.drop([0], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#state corpus \n",
    "texas = open_and_flatten('TXTfiles/texas')  \n",
    "\n",
    "#create a nested list\n",
    "a = ['texas'] \n",
    "a.append(texas) \n",
    "\n",
    "#create a alaska dataframe\n",
    "state_corpi = pd.DataFrame(a, columns=[\"corpus\"]) \n",
    "state_corpi['state'] = \"texas\" \n",
    "state_corpi = state_corpi.drop([0], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>corpus</th>\n",
       "      <th>state</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'elementary', 'essential', 'knowledge', 'skill...</td>\n",
       "      <td>texas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>'topic', 'arrangements', 'next', 'generation',...</td>\n",
       "      <td>ngss</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              corpus  state\n",
       "1  'elementary', 'essential', 'knowledge', 'skill...  texas\n",
       "1  'topic', 'arrangements', 'next', 'generation',...   ngss"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#join states & ngss\n",
    "standards_corpi = pd.concat([state_corpi, ngss_corpi], axis=0)  \n",
    "  \n",
    "\n",
    "#clearn corpus column\n",
    "standards_corpi['corpus'] = standards_corpi['corpus'].astype(str)  \n",
    "standards_corpi['corpus'] = standards_corpi['corpus'].apply(lambda x: \n",
    "                                                            x.strip(\"[\")\n",
    "                                                            .strip(\"]\"))\n",
    "\n",
    "standards_corpi.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I chose the Arizona state standards because they are heavily aligned with the NGSS standards so should produce a high number of similar points."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#turn text into a Scattertext Corpus \n",
    "nlp = spacy.load('en')\n",
    "corpus = st.CorpusFromPandas(standards_corpi, \n",
    "                            category_col='state', \n",
    "                            text_col='corpus', \n",
    "                            nlp=nlp).build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus Unique Words:\n",
      "- dcis\n",
      "- crosscutting\n",
      "- abstractly\n",
      "- testable\n",
      "- geosphere\n",
      "- nonliving\n",
      "- progresses\n",
      "- hydrosphere\n",
      "- delimiting\n",
      "- quantitatively\n"
     ]
    }
   ],
   "source": [
    "print(\"Corpus Unique Words:\")\n",
    "word = list(corpus.get_scaled_f_scores_vs_background().index[:10]) \n",
    "for w in word: \n",
    "    print(\"-\", w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Texas:\n",
      "- international\n",
      "- concepts student\n",
      "- student knows\n",
      "- adopted\n",
      "- effective\n",
      "- knows\n",
      "- student expected\n",
      "- update\n",
      "- provisions\n",
      "- investigation reasoning\n",
      "- international baccalaureate\n",
      "- baccalaureate\n",
      "- scientific investigation\n",
      "- adopted effective\n",
      "- provisions adopted\n",
      "- source provisions\n",
      "- recommended\n",
      "- laboratory field\n",
      "- critical thinking\n",
      "- field investigations\n"
     ]
    }
   ],
   "source": [
    "#words most associated with aligned states \n",
    "print(\"Texas:\")\n",
    "term_freq_df = corpus.get_term_freq_df()\n",
    "term_freq_df['Texas'] = corpus.get_scaled_f_scores('texas')\n",
    "al = list(term_freq_df.sort_values(by='Texas', ascending=False) \n",
    "          .index[:20])\n",
    "for a in al: \n",
    "    print(\"-\", a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NGSS:\n",
      "- engineering\n",
      "- builds\n",
      "- expectations\n",
      "- performance expectations\n",
      "- framework education\n",
      "- core ideas\n",
      "- clarification statement\n",
      "- experiences progresses\n",
      "- disciplinary core\n",
      "- assessment\n",
      "- clarification\n",
      "- boundary\n",
      "- could include\n",
      "- progresses\n",
      "- performance\n",
      "- statement\n",
      "- boundary assessment\n",
      "- assessment boundary\n",
      "- integrate\n",
      "- constructing\n"
     ]
    }
   ],
   "source": [
    "#words most associated with aligned states \n",
    "print(\"NGSS:\")\n",
    "term_freq_df = corpus.get_term_freq_df()\n",
    "term_freq_df['NGSS'] = corpus.get_scaled_f_scores('ngss')\n",
    "al = list(term_freq_df.sort_values(by='NGSS', ascending=False) \n",
    "          .index[:20])\n",
    "for a in al: \n",
    "    print(\"-\", a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1611141"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "html = st.produce_scattertext_explorer(corpus, \n",
    "                                       category='ngss',\n",
    "                                       category_name='NGSS',\n",
    "                                       not_category_name='Texas',\n",
    "                                       width_in_pixels=1000)\n",
    "open(\"TX_NGSS_Visualization.html\", 'wb').write(html.encode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}