TFIDF_Vectors_Final_Anol&Venant.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataframe for tf:\n",
      "            dc0.txt   dc1.txt   dc2.txt\n",
      "A          0.035714  0.035714  0.000000\n",
      "man        0.071429  0.142857  0.000000\n",
      "is         0.071429  0.035714  0.000000\n",
      "worth      0.035714  0.000000  0.000000\n",
      "a          0.142857  0.285714  0.035714\n",
      "price.The  0.035714  0.000000  0.000000\n",
      "real       0.035714  0.000000  0.000000\n",
      "man        0.071429  0.142857  0.000000\n",
      "is         0.071429  0.035714  0.000000\n",
      "here       0.035714  0.035714  0.000000\n",
      "roy        0.035714  0.035714  0.000000\n",
      "A          0.035714  0.035714  0.000000\n",
      "man        0.071429  0.142857  0.000000\n",
      "man        0.071429  0.142857  0.000000\n",
      "is         0.071429  0.035714  0.000000\n",
      "a          0.142857  0.285714  0.035714\n",
      "man        0.071429  0.142857  0.000000\n",
      "which      0.000000  0.035714  0.000000\n",
      "have       0.000000  0.035714  0.000000\n",
      "a          0.142857  0.285714  0.035714\n",
      "wife       0.000000  0.035714  0.000000\n",
      "without    0.000000  0.035714  0.000000\n",
      "a          0.142857  0.285714  0.035714\n",
      "man.here   0.000000  0.035714  0.000000\n",
      "roy        0.035714  0.035714  0.000000\n",
      "respect    0.000000  0.000000  0.035714\n",
      "the        0.000000  0.000000  0.035714\n",
      "parents    0.000000  0.000000  0.035714\n",
      "\n",
      "\n",
      "\n",
      "Dataframe for idf:\n",
      "            dc0.txt   dc1.txt   dc2.txt\n",
      "A          0.405465  0.405465  0.405465\n",
      "man        0.405465  0.405465  0.405465\n",
      "is         0.405465  0.405465  0.405465\n",
      "worth      1.098612  1.098612  1.098612\n",
      "a          0.000000  0.000000  0.000000\n",
      "price.The  1.098612  1.098612  1.098612\n",
      "real       1.098612  1.098612  1.098612\n",
      "man        0.405465  0.405465  0.405465\n",
      "is         0.405465  0.405465  0.405465\n",
      "here       0.405465  0.405465  0.405465\n",
      "roy        0.405465  0.405465  0.405465\n",
      "A          0.405465  0.405465  0.405465\n",
      "man        0.405465  0.405465  0.405465\n",
      "man        0.405465  0.405465  0.405465\n",
      "is         0.405465  0.405465  0.405465\n",
      "a          0.000000  0.000000  0.000000\n",
      "man        0.405465  0.405465  0.405465\n",
      "which      1.098612  1.098612  1.098612\n",
      "have       1.098612  1.098612  1.098612\n",
      "a          0.000000  0.000000  0.000000\n",
      "wife       1.098612  1.098612  1.098612\n",
      "without    1.098612  1.098612  1.098612\n",
      "a          0.000000  0.000000  0.000000\n",
      "man.here   1.098612  1.098612  1.098612\n",
      "roy        0.405465  0.405465  0.405465\n",
      "respect    1.098612  1.098612  1.098612\n",
      "the        1.098612  1.098612  1.098612\n",
      "parents    1.098612  1.098612  1.098612\n",
      "\n",
      "\n",
      "\n",
      "Dataframe for tf_idf:\n",
      "            dc0.txt   dc1.txt   dc2.txt\n",
      "A          0.014481  0.014481  0.000000\n",
      "man        0.028962  0.057924  0.000000\n",
      "is         0.028962  0.014481  0.000000\n",
      "worth      0.039236  0.000000  0.000000\n",
      "a          0.000000  0.000000  0.000000\n",
      "price.The  0.039236  0.000000  0.000000\n",
      "real       0.039236  0.000000  0.000000\n",
      "man        0.028962  0.057924  0.000000\n",
      "is         0.028962  0.014481  0.000000\n",
      "here       0.014481  0.014481  0.000000\n",
      "roy        0.014481  0.014481  0.000000\n",
      "A          0.014481  0.014481  0.000000\n",
      "man        0.028962  0.057924  0.000000\n",
      "man        0.028962  0.057924  0.000000\n",
      "is         0.028962  0.014481  0.000000\n",
      "a          0.000000  0.000000  0.000000\n",
      "man        0.028962  0.057924  0.000000\n",
      "which      0.000000  0.039236  0.000000\n",
      "have       0.000000  0.039236  0.000000\n",
      "a          0.000000  0.000000  0.000000\n",
      "wife       0.000000  0.039236  0.000000\n",
      "without    0.000000  0.039236  0.000000\n",
      "a          0.000000  0.000000  0.000000\n",
      "man.here   0.000000  0.039236  0.000000\n",
      "roy        0.014481  0.014481  0.000000\n",
      "respect    0.000000  0.000000  0.039236\n",
      "the        0.000000  0.000000  0.039236\n",
      "parents    0.000000  0.000000  0.039236\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# include modules\n",
    "from os import listdir #here we only want to import listdir\"list directory\",\n",
    "#not the entire os module for the the sake of speed and memory \n",
    "from os.path import isfile, join #same here but what is isfile???\n",
    "import math #here we are impoting math means all formulations and stuffs.\n",
    "import re # regular expression\n",
    "from pandas import DataFrame\n",
    "\n",
    "# fetching documents\n",
    "mypath = \"C:/Users/Venant/Desktop/basicscorpus\"\n",
    "\n",
    "filenames = [f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
    "\n",
    "docs = []\n",
    "for file in filenames:\n",
    "    file_path = mypath + '/' + file \n",
    "    with open(file_path) as f:\n",
    "        str = ' '.join(f.readlines())\n",
    "        docs.append(str)\n",
    "\n",
    "tf_list = [] \n",
    "idf_list = []\n",
    "\n",
    "# calculations\n",
    "words = []\n",
    "for doc in docs:\n",
    "    words += doc.split()\n",
    "\n",
    "# calculate tf\n",
    "for word in words:\n",
    "    lst = []\n",
    "    for doc in docs:\n",
    "        word_count = sum(1 for match in re.finditer(word, doc))\n",
    "        total_word_count = len(words)\n",
    "        tf = word_count / total_word_count\n",
    "        lst.append(tf)\n",
    "    tf_list.append(lst)\n",
    "    \n",
    "df_tf = DataFrame(tf_list)\n",
    "print('Dataframe for tf:')\n",
    "df_tf.index = words\n",
    "df_tf.columns = filenames\n",
    "print(df_tf)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "# calculate idf\n",
    "\n",
    "def get_doc_count(word, docs):\n",
    "    doc_count = 0\n",
    "    for doc in docs:\n",
    "        if word in doc:\n",
    "            doc_count += 1\n",
    "    return doc_count\n",
    "\n",
    "for word in words:\n",
    "    lst = []\n",
    "    total_doc_count = len(docs)\n",
    "    doc_count = get_doc_count(word, docs)\n",
    "    for doc in docs:\n",
    "        idf = math.log(total_doc_count / doc_count)\n",
    "        #idf = math.log((total_doc_count / doc_count) , 10)\n",
    "        lst.append(idf)\n",
    "    idf_list.append(lst)\n",
    "\n",
    "df_idf = DataFrame(idf_list)\n",
    "print('Dataframe for idf:')\n",
    "df_idf.index = words\n",
    "df_idf.columns = filenames\n",
    "print(df_idf)\n",
    "print('\\n\\n')\n",
    "\n",
    "\n",
    "# calculate tf_idf\n",
    "df_tf_idf = DataFrame(df_tf.values*df_idf.values, columns=filenames, index=words)\n",
    "print('Dataframe for tf_idf:')\n",
    "print(df_tf_idf)\n",
    "print('\\n\\n') #adding double space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}