Data_Extraction_and_Text_Analysis_for_Blackcoffer_company.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/gdrive')\n",
    "%cd '/gdrive/MyDrive/project/Data_Extraction_and_NLP/TestAssignment'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import necessary pacakages\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import os\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#read the url file into the pandas object\n",
    "df = pd.read_excel('Input.xlsx')\n",
    "\n",
    "#loop throgh each row in the df\n",
    "for index, row in df.iterrows():\n",
    "  url = row['URL']\n",
    "  url_id = row['URL_ID']\n",
    "\n",
    "  # make a request to url\n",
    "  header = {'User-Agent': \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36\"}\n",
    "  try:\n",
    "    response = requests.get(url,headers=header)\n",
    "  except:\n",
    "    print(\"can't get response of {}\".format(url_id))\n",
    "\n",
    "  #create a beautifulsoup object\n",
    "  try:\n",
    "    soup = BeautifulSoup(response.content, 'html.parser')\n",
    "  except:\n",
    "    print(\"can't get page of {}\".format(url_id))\n",
    "  #find title\n",
    "  try:\n",
    "    title = soup.find('h1').get_text()\n",
    "  except:\n",
    "    print(\"can't get title of {}\".format(url_id))\n",
    "    continue\n",
    "  #find text\n",
    "  article = \"\"\n",
    "  try:\n",
    "    for p in soup.find_all('p'):\n",
    "      article += p.get_text()\n",
    "  except:\n",
    "    print(\"can't get text of {}\".format(url_id))\n",
    "\n",
    "  #write title and text to the file\n",
    "  file_name = '/gdrive/MyDrive/project/Data_Extraction_and_NLP/TestAssignment/TitleText/' + str(url_id) + '.txt'\n",
    "  with open(file_name, 'w') as file:\n",
    "    file.write(title + '\\n' + article)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Directories\n",
    "text_dir = \"/gdrive/MyDrive/project/Data_Extraction_and_NLP/TestAssignment/TitleText\"\n",
    "stopwords_dir = \"/gdrive/MyDrive/project/Data_Extraction_and_NLP/TestAssignment/StopWords\"\n",
    "sentment_dir = \"/gdrive/MyDrive/project/Data_Extraction_and_NLP/TestAssignment/MasterDictionary\"\n",
    "\n",
    "# load all stop wors from the stopwords directory and store in the set variable\n",
    "stop_words = set()\n",
    "for files in os.listdir(stopwords_dir):\n",
    "  with open(os.path.join(stopwords_dir,files),'r',encoding='ISO-8859-1') as f:\n",
    "    stop_words.update(set(f.read().splitlines()))\n",
    "\n",
    "# load all text files  from the  directory and store in a list(docs)\n",
    "docs = []\n",
    "for text_file in os.listdir(text_dir):\n",
    "  with open(os.path.join(text_dir,text_file),'r') as f:\n",
    "    text = f.read()\n",
    "#tokenize the given text file\n",
    "    words = word_tokenize(text)\n",
    "# remove the stop words from the tokens\n",
    "    filtered_text = [word for word in words if word.lower() not in stop_words]\n",
    "# add each filtered tokens of each file into a list\n",
    "    docs.append(filtered_text)\n",
    "\n",
    "\n",
    "\n",
    "# store positive, Negative words from the directory\n",
    "pos=set()\n",
    "neg=set()\n",
    "\n",
    "for files in os.listdir(sentment_dir):\n",
    "  if files =='positive-words.txt':\n",
    "    with open(os.path.join(sentment_dir,files),'r',encoding='ISO-8859-1') as f:\n",
    "      pos.update(f.read().splitlines())\n",
    "  else:\n",
    "    with open(os.path.join(sentment_dir,files),'r',encoding='ISO-8859-1') as f:\n",
    "      neg.update(f.read().splitlines())\n",
    "\n",
    "# now collect the positive  and negative words from each file\n",
    "# calculate the scores from the positive and negative words \n",
    "positive_words = []\n",
    "Negative_words =[]\n",
    "positive_score = []\n",
    "negative_score = []\n",
    "polarity_score = []\n",
    "subjectivity_score = []\n",
    "\n",
    "#iterate through the list of docs\n",
    "for i in range(len(docs)):\n",
    "  positive_words.append([word for word in docs[i] if word.lower() in pos])\n",
    "  Negative_words.append([word for word in docs[i] if word.lower() in neg])\n",
    "  positive_score.append(len(positive_words[i]))\n",
    "  negative_score.append(len(Negative_words[i]))\n",
    "  polarity_score.append((positive_score[i] - negative_score[i]) / ((positive_score[i] + negative_score[i]) + 0.000001))\n",
    "  subjectivity_score.append((positive_score[i] + negative_score[i]) / ((len(docs[i])) + 0.000001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average Sentence Length = the number of words / the number of sentences\n",
    "# Percentage of Complex words = the number of complex words / the number of words \n",
    "# Fog Index = 0.4 * (Average Sentence Length + Percentage of Complex words)\n",
    "\n",
    "avg_sentence_length = []\n",
    "Percentage_of_Complex_words  =  []\n",
    "Fog_Index = []\n",
    "complex_word_count =  []\n",
    "avg_syllable_word_count =[]\n",
    "\n",
    "stopwords = set(stopwords.words('english'))\n",
    "def measure(file):\n",
    "  with open(os.path.join(text_dir, file),'r') as f:\n",
    "    text = f.read()\n",
    "# remove punctuations \n",
    "    text = re.sub(r'[^\\w\\s.]','',text)\n",
    "# split the given text file into sentences\n",
    "    sentences = text.split('.')\n",
    "# total number of sentences in a file\n",
    "    num_sentences = len(sentences)\n",
    "# total words in the file\n",
    "    words = [word  for word in text.split() if word.lower() not in stopwords ]\n",
    "    num_words = len(words)\n",
    " \n",
    "# complex words having syllable count is greater than 2\n",
    "# Complex words are words in the text that contain more than two syllables.\n",
    "    complex_words = []\n",
    "    for word in words:\n",
    "      vowels = 'aeiou'\n",
    "      syllable_count_word = sum( 1 for letter in word if letter.lower() in vowels)\n",
    "      if syllable_count_word > 2:\n",
    "        complex_words.append(word)\n",
    "\n",
    "# Syllable Count Per Word\n",
    "# We count the number of Syllables in each word of the text by counting the vowels present in each word.\n",
    "#  We also handle some exceptions like words ending with \"es\",\"ed\" by not counting them as a syllable.\n",
    "    syllable_count = 0\n",
    "    syllable_words =[]\n",
    "    for word in words:\n",
    "      if word.endswith('es'):\n",
    "        word = word[:-2]\n",
    "      elif word.endswith('ed'):\n",
    "        word = word[:-2]\n",
    "      vowels = 'aeiou'\n",
    "      syllable_count_word = sum( 1 for letter in word if letter.lower() in vowels)\n",
    "      if syllable_count_word >= 1:\n",
    "        syllable_words.append(word)\n",
    "        syllable_count += syllable_count_word\n",
    "\n",
    "\n",
    "    avg_sentence_len = num_words / num_sentences\n",
    "    avg_syllable_word_count = syllable_count / len(syllable_words)\n",
    "    Percent_Complex_words  =  len(complex_words) / num_words\n",
    "    Fog_Index = 0.4 * (avg_sentence_len + Percent_Complex_words)\n",
    "\n",
    "    return avg_sentence_len, Percent_Complex_words, Fog_Index, len(complex_words),avg_syllable_word_count\n",
    "\n",
    "# iterate through each file or doc\n",
    "for file in os.listdir(text_dir):\n",
    "  x,y,z,a,b = measure(file)\n",
    "  avg_sentence_length.append(x)\n",
    "  Percentage_of_Complex_words.append(y)\n",
    "  Fog_Index.append(z)\n",
    "  complex_word_count.append(a)\n",
    "  avg_syllable_word_count.append(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Word Count and Average Word Length Sum of the total number of characters in each word/Total number of words\n",
    "# We count the total cleaned words present in the text by \n",
    "# removing the stop words (using stopwords class of nltk package).\n",
    "# removing any punctuations like ? ! , . from the word before counting.\n",
    "\n",
    "def cleaned_words(file):\n",
    "  with open(os.path.join(text_dir,file), 'r') as f:\n",
    "    text = f.read()\n",
    "    text = re.sub(r'[^\\w\\s]', '' , text)\n",
    "    words = [word  for word in text.split() if word.lower() not in stopwords]\n",
    "    length = sum(len(word) for word in words)\n",
    "    average_word_length = length / len(words)\n",
    "  return len(words),average_word_length\n",
    "\n",
    "word_count = []\n",
    "average_word_length = []\n",
    "for file in os.listdir(text_dir):\n",
    "  x, y = cleaned_words(file)\n",
    "  word_count.append(x)\n",
    "  average_word_length.append(y)\n",
    "\n",
    "\n",
    "# To calculate Personal Pronouns mentioned in the text, we use regex to find \n",
    "# the counts of the words - “I,” “we,” “my,” “ours,” and “us”. Special care is taken\n",
    "#  so that the country name US is not included in the list.\n",
    "def count_personal_pronouns(file):\n",
    "  with open(os.path.join(text_dir,file), 'r') as f:\n",
    "    text = f.read()\n",
    "    personal_pronouns = [\"I\", \"we\", \"my\", \"ours\", \"us\"]\n",
    "    count = 0\n",
    "    for pronoun in personal_pronouns:\n",
    "      count += len(re.findall(r\"\\b\" + pronoun + r\"\\b\", text)) # \\b is used to match word boundaries\n",
    "  return count\n",
    "\n",
    "pp_count = []\n",
    "for file in os.listdir(text_dir):\n",
    "  x = count_personal_pronouns(file)\n",
    "  pp_count.append(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "output_df = pd.read_excel('Output Data Structure.xlsx')\n",
    "\n",
    "# URL_ID 44 ,57, 144 does not exists i,e. page does not exist, throughs 404 error\n",
    "# so we are going to drop these rows from the table\n",
    "output_df.drop([44-37,57-37,144-37], axis = 0, inplace=True)\n",
    "\n",
    "# These are the required parameters \n",
    "variables = [positive_score,\n",
    "            negative_score,\n",
    "            polarity_score,\n",
    "            subjectivity_score,\n",
    "            avg_sentence_length,\n",
    "            Percentage_of_Complex_words,\n",
    "            Fog_Index,\n",
    "            avg_sentence_length,\n",
    "            complex_word_count,\n",
    "            word_count,\n",
    "            avg_syllable_word_count,\n",
    "            pp_count,\n",
    "            average_word_length]\n",
    "\n",
    "# write the values to the dataframe\n",
    "for i, var in enumerate(variables):\n",
    "  output_df.iloc[:,i+2] = var\n",
    "\n",
    "#now save the dataframe to the disk\n",
    "output_df.to_csv('Output_Data.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}