Quote_List_Builder.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "learn-env",
   "display_name": "learn-env"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import tweepy\n",
    "import numpy as np\n",
    "from nltk import sent_tokenize\n",
    "import re\n",
    "import json\n",
    "\n",
    "\n",
    "def get_Guten(url):\n",
    "    # retrieve the source text\n",
    "    r = requests.get(url)\n",
    "    r.encoding = 'utf-8'\n",
    "    text = r.text\n",
    "    return text\n",
    "\n",
    "def get_text(path):\n",
    "    f = open(path, 'r', encoding='utf8')\n",
    "    text = f.read()\n",
    "    f.close()\n",
    "    return text\n",
    "\n",
    "def clean_text(text):\n",
    "    with open(\"data/replace_chars.json\") as f:\n",
    "        replace_these = json.load(f)\n",
    "        for k in replace_these.keys():\n",
    "            text = text.replace(k, replace_these[k])\n",
    "    text = text.replace('\\xad ', '').replace('\\x0c', '')\n",
    "    return text\n",
    "\n",
    "def make_into_quotes_guten(text, source):\n",
    "    # make a list of quotes and clean them up\n",
    "    quotes = sent_tokenize(text)\n",
    "    # remove unnecessary spaces\n",
    "    quotes = [x.strip() for x in quotes]\n",
    "    # remove empty quotes\n",
    "    quotes = list(filter(None, quotes))\n",
    "    # cut out very short ones as they often have no real meaning\n",
    "    quotes = [x for x in quotes if len(x) > 15]\n",
    "    # remove the titles of sections & citation-type stuff\n",
    "    quotes = [x for x in quotes if not x.isupper()]\n",
    "    quotes = [x for x in quotes if not x.replace('the', '').replace('of', '').replace('and', '').replace('II', '').istitle()]\n",
    "    quotes = [x for x in quotes if not set('Werke').issubset(x)]\n",
    "    quotes = [x for x in quotes if not set('Footnote').issubset(x)]\n",
    "    # remove oddities\n",
    "    quotes = [x for x in quotes if x[0].isupper()]\n",
    "    quotes = [x.replace('.', '') for x in quotes]\n",
    "    quotes = [x for x in quotes if not x[-1].isupper()]\n",
    "    # add the source\n",
    "    quotes = [x+'\\n- '+ source for x in quotes]\n",
    "    return quotes\n",
    "\n",
    "def make_into_quotes_pdf(text, source):\n",
    "    # make the text into a list\n",
    "    quotes = sent_tokenize(text)\n",
    "    # remove unnecessary spaces\n",
    "    quotes = [x.strip() for x in quotes]\n",
    "    # remove empty quotes\n",
    "    quotes = list(filter(None, quotes))\n",
    "    # cut out very short ones as they often have no real meaning\n",
    "    quotes = [x for x in quotes if len(x) > 15]\n",
    "    # remove the titles of sections & citation-type stuff\n",
    "    quotes = [x for x in quotes if not x.isupper()]\n",
    "    quotes = [x for x in quotes if not x.replace('the', '').replace('of', '').replace('and', '').replace('II', '').istitle()]\n",
    "    quotes = [x for x in quotes if not set('Werke').issubset(x)]\n",
    "    # this looks at all quotes and removes headers/footers/page numbers that are sometimes in the text accidentally\n",
    "    holding = []\n",
    "    for quote in quotes:\n",
    "        for word in quote.split(' '):\n",
    "            if word.isupper() and len(word) > 2 and word != 'A' and word != 'OF':\n",
    "                quote = quote.replace(word, '')\n",
    "        quote = re.sub('[1234567890]', '', quote).replace(' s ', ' ').replace(' S ', ' ').replace('OF', '').replace(' ) ', '').replace(' ( ',                   '').replace(' , ', '').replace('- ', '-').replace('  ', ' ').replace('  ', ' ').replace('  ', ' ')\n",
    "        holding.append(quote)\n",
    "    # remove oddities\n",
    "    quotes = [x for x in holding if x[0].isupper()]\n",
    "    quotes = [x.replace('.', '') for x in quotes]\n",
    "    quotes = [x for x in quotes if not x[-1].isupper()]\n",
    "    quotes = [x for x in quotes if not set('~').issubset(x)]\n",
    "    quotes = [x for x in quotes if not set('=').issubset(x)]\n",
    "    # add the source\n",
    "    quotes = [x+'\\n- '+ source for x in quotes]\n",
    "    return quotes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # import different texts, cut out their front and end matter\n",
    "hop1 = clean_text(get_Guten('http://www.gutenberg.org/files/51635/51635-0.txt'))[10545:-34150]\n",
    "hop2 = clean_text(get_Guten('http://www.gutenberg.org/files/51636/51636-0.txt'))[4489:-42865]\n",
    "hop3 = clean_text(get_Guten('http://www.gutenberg.org/files/58169/58169-0.txt'))[10068:-125524]\n",
    "enc_logic = clean_text(get_Guten('http://www.gutenberg.org/files/55108/55108-0.txt'))[36755:-134712]\n",
    "phil_of_nature = clean_text(get_text('.\\data\\Phil_of_Nature.txt'))[500403:-278448]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(['Here, instead of surveying the process, as we do in history, from the outside, we see the movement of thought clearly defined in its native medium\\n- EnL',\n",
       "  'The thought, which is genuine and self-supporting, must be intrinsically concrete; it must be an Idea; and when it is viewed in the whole of its universality, it is the Idea, or the Absolute\\n- EnL',\n",
       "  'The science of this Idea must form a system\\n- EnL',\n",
       "  'For the truth is concrete; that is, whilst it gives a bond and principle of unity, it also possesses an internal source of development\\n- EnL',\n",
       "  'Truth, then, is only possible as a universe or totality of thought; and the freedom of the whole, as well as the necessity of the several sub-divisions, which it implies, are only possible when these are discriminated and defined\\n- EnL',\n",
       "  'Unless it is a system, a philosophy is not a scientific production\\n- EnL',\n",
       "  'Unsystematic philosophising can only be expected to give expression to personal peculiarities of mind, and has no principle for the regulation of its contents\\n- EnL',\n",
       "  'Apart from their interdependence and organic union, the truths of philosophy are valueless, and must then be treated as baseless hypotheses, or personal convictions\\n- EnL',\n",
       "  'Yet many philosophical treatises confine themselves to such an exposition of the opinions and sentiments of the author\\n- EnL',\n",
       "  'The term system is often misunderstood\\n- EnL'],\n",
       " 18316)"
      ]
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "# turn these texts into quotes and assemble a list\n",
    "hop1_quotes = make_into_quotes_guten(hop1, 'HoP 1')\n",
    "hop2_quotes = make_into_quotes_guten(hop2, 'HoP 2')\n",
    "hop3_quotes = make_into_quotes_guten(hop3, 'HoP 3')\n",
    "enc_logic_quotes = make_into_quotes_guten(enc_logic, 'EnL')\n",
    "pon_quotes = make_into_quotes_pdf(phil_of_nature, 'PoN')\n",
    "\n",
    "master_q = hop1_quotes + hop2_quotes + hop3_quotes + enc_logic_quotes + pon_quotes\n",
    "\n",
    "# preview the quote list to see if there are any abberations\n",
    "random_range_start = np.random.randint(0, len(master_q))\n",
    "master_q[random_range_start:random_range_start + 10], len(master_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the original set of quotes and prepare it for merging\n",
    "old_quotes = pd.read_csv('.\\data\\Original_Quote_sheet.csv')\n",
    "old_quotes = old_quotes.drop('Unnamed: 0', axis=1)\n",
    "old_quotes = old_quotes.rename(columns={'Select one from each column':'quotes'})\n",
    "old_quotes = old_quotes.iloc[3:]\n",
    "old_quotes['quotes'] = old_quotes['quotes'].str.capitalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(                                                  quotes  length\n",
       " 15058  This is correct, if it means that a man's cond...     184\n",
       " 15060  Self-relation in Essence is the form of Identi...     139\n",
       " 15061  They are both the same abstraction,--self-rela...      56\n",
       " 15062  The unintelligence of sense, to take everythin...     203\n",
       " 15063  This identity, as it has descended from Being,...     174\n",
       " 15064  This external Being, if taken in separation fr...     109\n",
       " 15065                But that turns out a mistake\\n- EnL      34\n",
       " 15066  Because Essence is Being-in-self, it is essent...     147\n",
       " 15067  Consequently, it has the unessential as its ow...      91\n",
       " 15069  The sphere of Essence thus turns out to be a s...     105,\n",
       " 15889,\n",
       " 18714)"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "# turn the list into a dataframe and weed out untweetabley-long quotes\n",
    "quote_df = pd.DataFrame(master_q, columns=['quotes'])\n",
    "quote_df = old_quotes.append(quote_df)\n",
    "quote_df['length'] = quote_df['quotes'].str.len()\n",
    "quote_tweetable = quote_df.loc[quote_df['length'] <= 280].copy()\n",
    "\n",
    "# preview again, see how many we have\n",
    "quote_tweetable.iloc[random_range_start:random_range_start + 10], len(quote_tweetable), len(quote_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export csv for use by tweeter program\n",
    "\n",
    "DO NOT RUN\n",
    "quote_tweetable.to_csv('Quote List.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ]
}