nlp_text_analysis.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VybcseieSTRZ","outputId":"f7e41cd6-a21b-45dc-dfb8-b0adafe7f75d","executionInfo":{"status":"ok","timestamp":1717056391360,"user_tz":-330,"elapsed":19780,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting goose3[all]\n","  Downloading goose3-3.1.19-py3-none-any.whl (113 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.4/113.4 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hCollecting textstat\n","  Downloading textstat-0.7.3-py3-none-any.whl (105 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.1/105.1 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (2.31.0)\n","Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (9.4.0)\n","Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (4.9.4)\n","Collecting cssselect (from goose3[all])\n","  Downloading cssselect-1.2.0-py2.py3-none-any.whl (18 kB)\n","Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (4.12.3)\n","Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (2.8.2)\n","Collecting langdetect (from goose3[all])\n","  Downloading langdetect-1.0.9.tar.gz (981 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m981.5/981.5 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Collecting pyahocorasick (from goose3[all])\n","  Downloading pyahocorasick-2.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (110 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.7/110.7 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: jieba in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (0.42.1)\n","Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from goose3[all]) (3.8.1)\n","Collecting pyphen (from textstat)\n","  Downloading pyphen-0.15.0-py3-none-any.whl (2.1 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.1/2.1 MB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->goose3[all]) (2.5)\n","Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->goose3[all]) (1.16.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->goose3[all]) (8.1.7)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk->goose3[all]) (1.4.2)\n","Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk->goose3[all]) (2024.5.15)\n","Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk->goose3[all]) (4.66.4)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->goose3[all]) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->goose3[all]) (3.7)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->goose3[all]) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->goose3[all]) (2024.2.2)\n","Building wheels for collected packages: langdetect\n","  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993227 sha256=e5cfbf6580cf1e152577623f7442b1d4a2fe98cb1c9a9ecab4494b50f279ecc7\n","  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n","Successfully built langdetect\n","Installing collected packages: pyphen, pyahocorasick, langdetect, cssselect, textstat, goose3\n","Successfully installed cssselect-1.2.0 goose3-3.1.19 langdetect-1.0.9 pyahocorasick-2.1.0 pyphen-0.15.0 textstat-0.7.3\n"]}],"source":["!pip install goose3[all] textstat"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Blg4hU98STRh","outputId":"76327ed3-0d4d-4d26-cc8b-61e4acc1f80e","executionInfo":{"status":"ok","timestamp":1717056445754,"user_tz":-330,"elapsed":54397,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading collection 'all'\n","[nltk_data]    | \n","[nltk_data]    | Downloading package abc to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/abc.zip.\n","[nltk_data]    | Downloading package alpino to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/alpino.zip.\n","[nltk_data]    | Downloading package averaged_perceptron_tagger to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n","[nltk_data]    | Downloading package averaged_perceptron_tagger_ru to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping\n","[nltk_data]    |       taggers/averaged_perceptron_tagger_ru.zip.\n","[nltk_data]    | Downloading package basque_grammars to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping grammars/basque_grammars.zip.\n","[nltk_data]    | Downloading package bcp47 to /root/nltk_data...\n","[nltk_data]    | Downloading package biocreative_ppi to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/biocreative_ppi.zip.\n","[nltk_data]    | Downloading package bllip_wsj_no_aux to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping models/bllip_wsj_no_aux.zip.\n","[nltk_data]    | Downloading package book_grammars to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n","[nltk_data]    | Downloading package brown to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/brown.zip.\n","[nltk_data]    | Downloading package brown_tei to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/brown_tei.zip.\n","[nltk_data]    | Downloading package cess_cat to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/cess_cat.zip.\n","[nltk_data]    | Downloading package cess_esp to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/cess_esp.zip.\n","[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/chat80.zip.\n","[nltk_data]    | Downloading package city_database to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/city_database.zip.\n","[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/cmudict.zip.\n","[nltk_data]    | Downloading package comparative_sentences to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/comparative_sentences.zip.\n","[nltk_data]    | Downloading package comtrans to /root/nltk_data...\n","[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/conll2000.zip.\n","[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/conll2002.zip.\n","[nltk_data]    | Downloading package conll2007 to /root/nltk_data...\n","[nltk_data]    | Downloading package crubadan to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/crubadan.zip.\n","[nltk_data]    | Downloading package dependency_treebank to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n","[nltk_data]    | Downloading package dolch to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/dolch.zip.\n","[nltk_data]    | Downloading package europarl_raw to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/europarl_raw.zip.\n","[nltk_data]    | Downloading package extended_omw to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    | Downloading package floresta to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/floresta.zip.\n","[nltk_data]    | Downloading package framenet_v15 to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/framenet_v15.zip.\n","[nltk_data]    | Downloading package framenet_v17 to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/framenet_v17.zip.\n","[nltk_data]    | Downloading package gazetteers to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/gazetteers.zip.\n","[nltk_data]    | Downloading package genesis to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/genesis.zip.\n","[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n","[nltk_data]    | Downloading package ieer to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/ieer.zip.\n","[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/inaugural.zip.\n","[nltk_data]    | Downloading package indian to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/indian.zip.\n","[nltk_data]    | Downloading package jeita to /root/nltk_data...\n","[nltk_data]    | Downloading package kimmo to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/kimmo.zip.\n","[nltk_data]    | Downloading package knbc to /root/nltk_data...\n","[nltk_data]    | Downloading package large_grammars to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping grammars/large_grammars.zip.\n","[nltk_data]    | Downloading package lin_thesaurus to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/lin_thesaurus.zip.\n","[nltk_data]    | Downloading package mac_morpho to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/mac_morpho.zip.\n","[nltk_data]    | Downloading package machado to /root/nltk_data...\n","[nltk_data]    | Downloading package masc_tagged to /root/nltk_data...\n","[nltk_data]    | Downloading package maxent_ne_chunker to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n","[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n","[nltk_data]    | Downloading package moses_sample to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping models/moses_sample.zip.\n","[nltk_data]    | Downloading package movie_reviews to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n","[nltk_data]    | Downloading package mte_teip5 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/mte_teip5.zip.\n","[nltk_data]    | Downloading package mwa_ppdb to /root/nltk_data...\n","[nltk_data]    |   Unzipping misc/mwa_ppdb.zip.\n","[nltk_data]    | Downloading package names to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/names.zip.\n","[nltk_data]    | Downloading package nombank.1.0 to /root/nltk_data...\n","[nltk_data]    | Downloading package nonbreaking_prefixes to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/nonbreaking_prefixes.zip.\n","[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n","[nltk_data]    | Downloading package omw to /root/nltk_data...\n","[nltk_data]    | Downloading package omw-1.4 to /root/nltk_data...\n","[nltk_data]    | Downloading package opinion_lexicon to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/opinion_lexicon.zip.\n","[nltk_data]    | Downloading package panlex_swadesh to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    | Downloading package paradigms to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/paradigms.zip.\n","[nltk_data]    | Downloading package pe08 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/pe08.zip.\n","[nltk_data]    | Downloading package perluniprops to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping misc/perluniprops.zip.\n","[nltk_data]    | Downloading package pil to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/pil.zip.\n","[nltk_data]    | Downloading package pl196x to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/pl196x.zip.\n","[nltk_data]    | Downloading package porter_test to /root/nltk_data...\n","[nltk_data]    |   Unzipping stemmers/porter_test.zip.\n","[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/ppattach.zip.\n","[nltk_data]    | Downloading package problem_reports to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/problem_reports.zip.\n","[nltk_data]    | Downloading package product_reviews_1 to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/product_reviews_1.zip.\n","[nltk_data]    | Downloading package product_reviews_2 to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/product_reviews_2.zip.\n","[nltk_data]    | Downloading package propbank to /root/nltk_data...\n","[nltk_data]    | Downloading package pros_cons to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/pros_cons.zip.\n","[nltk_data]    | Downloading package ptb to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/ptb.zip.\n","[nltk_data]    | Downloading package punkt to /root/nltk_data...\n","[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n","[nltk_data]    | Downloading package qc to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/qc.zip.\n","[nltk_data]    | Downloading package reuters to /root/nltk_data...\n","[nltk_data]    | Downloading package rslp to /root/nltk_data...\n","[nltk_data]    |   Unzipping stemmers/rslp.zip.\n","[nltk_data]    | Downloading package rte to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/rte.zip.\n","[nltk_data]    | Downloading package sample_grammars to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping grammars/sample_grammars.zip.\n","[nltk_data]    | Downloading package semcor to /root/nltk_data...\n","[nltk_data]    | Downloading package senseval to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/senseval.zip.\n","[nltk_data]    | Downloading package sentence_polarity to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/sentence_polarity.zip.\n","[nltk_data]    | Downloading package sentiwordnet to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/sentiwordnet.zip.\n","[nltk_data]    | Downloading package shakespeare to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/shakespeare.zip.\n","[nltk_data]    | Downloading package sinica_treebank to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/sinica_treebank.zip.\n","[nltk_data]    | Downloading package smultron to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/smultron.zip.\n","[nltk_data]    | Downloading package snowball_data to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    | Downloading package spanish_grammars to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping grammars/spanish_grammars.zip.\n","[nltk_data]    | Downloading package state_union to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/state_union.zip.\n","[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/stopwords.zip.\n","[nltk_data]    | Downloading package subjectivity to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/subjectivity.zip.\n","[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/swadesh.zip.\n","[nltk_data]    | Downloading package switchboard to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/switchboard.zip.\n","[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n","[nltk_data]    |   Unzipping help/tagsets.zip.\n","[nltk_data]    | Downloading package timit to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/timit.zip.\n","[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/toolbox.zip.\n","[nltk_data]    | Downloading package treebank to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/treebank.zip.\n","[nltk_data]    | Downloading package twitter_samples to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/twitter_samples.zip.\n","[nltk_data]    | Downloading package udhr to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/udhr.zip.\n","[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/udhr2.zip.\n","[nltk_data]    | Downloading package unicode_samples to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n","[nltk_data]    | Downloading package universal_tagset to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n","[nltk_data]    | Downloading package universal_treebanks_v20 to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    | Downloading package vader_lexicon to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    | Downloading package verbnet to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/verbnet.zip.\n","[nltk_data]    | Downloading package verbnet3 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/verbnet3.zip.\n","[nltk_data]    | Downloading package webtext to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/webtext.zip.\n","[nltk_data]    | Downloading package wmt15_eval to /root/nltk_data...\n","[nltk_data]    |   Unzipping models/wmt15_eval.zip.\n","[nltk_data]    | Downloading package word2vec_sample to\n","[nltk_data]    |     /root/nltk_data...\n","[nltk_data]    |   Unzipping models/word2vec_sample.zip.\n","[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n","[nltk_data]    | Downloading package wordnet2021 to /root/nltk_data...\n","[nltk_data]    | Downloading package wordnet2022 to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/wordnet2022.zip.\n","[nltk_data]    | Downloading package wordnet31 to /root/nltk_data...\n","[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n","[nltk_data]    | Downloading package words to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/words.zip.\n","[nltk_data]    | Downloading package ycoe to /root/nltk_data...\n","[nltk_data]    |   Unzipping corpora/ycoe.zip.\n","[nltk_data]    | \n","[nltk_data]  Done downloading collection all\n","[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data]   Package stopwords is already up-to-date!\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":2}],"source":["import nltk\n","nltk.download('all')\n","nltk.download('stopwords')"]},{"cell_type":"code","execution_count":3,"metadata":{"id":"gWAEGI3xSTRj","executionInfo":{"status":"ok","timestamp":1717056445754,"user_tz":-330,"elapsed":9,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["import pandas as pd\n","import numpy as np\n","from goose3 import Goose\n","import textstat\n","from nltk.corpus import stopwords\n","from nltk.tokenize import word_tokenize\n","from string import punctuation\n","import re\n","import os"]},{"cell_type":"markdown","metadata":{"id":"v-GZFhGhSTRl"},"source":["# Data Extraction"]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"gxx3--BKSXK1","outputId":"0dcf1eee-e740-4c7c-9600-28fe77c3c1cd","executionInfo":{"status":"ok","timestamp":1717056471891,"user_tz":-330,"elapsed":26144,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":5,"metadata":{"id":"M49hsC0xSTRn","executionInfo":{"status":"ok","timestamp":1717056471891,"user_tz":-330,"elapsed":14,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["def data_extraction():\n","    data=pd.read_excel(\"/content/drive/MyDrive/Text_Analysis/Input.xlsx\")\n","    df=pd.DataFrame({\"title\":[],\"text\":[]})\n","    for i in range(len(data)):\n","        g = Goose({'browser_user_agent': 'Mozilla', 'parser_class':'soup','strict': False})\n","        article = g.extract(url=data.loc[i].URL)\n","        title=article.title\n","        text=article.cleaned_text.replace('\\n','')\n","        df.loc[i]=[title, text]\n","    final_data=pd.concat([data,df],axis=1)\n","    return final_data"]},{"cell_type":"markdown","metadata":{"id":"gY0rvhxWSTRo"},"source":["# Text Preprocessing and StopWords"]},{"cell_type":"code","execution_count":6,"metadata":{"id":"BAl6LmqZSTRp","executionInfo":{"status":"ok","timestamp":1717056471892,"user_tz":-330,"elapsed":13,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["def stop_words():\n","    #loading stopwords\n","    StopWords=[]\n","    path=r\"/content/drive/MyDrive/Text_Analysis/StopWords\"\n","    for file in os.listdir(path):\n","        file_path = os.path.join(path, file)\n","        with open(file_path,\"r\", encoding=\"latin-1\") as f:\n","            l=f.read().split('\\n')\n","            l=[val for val in l if val!='']\n","            StopWords+=l\n","    # cleaning stopwords\n","    for word in StopWords:\n","        if \" | \" in word:\n","            split_words=re.split(r' \\| ', word, flags=re.IGNORECASE)\n","            StopWords.extend(split_words)\n","            StopWords.remove(word)\n","    StopWords=[val for val in list(set(StopWords)) if len(val)<=30]\n","    return StopWords"]},{"cell_type":"code","execution_count":7,"metadata":{"id":"e1a1zUukSTRq","executionInfo":{"status":"ok","timestamp":1717056471892,"user_tz":-330,"elapsed":12,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["def positive_dict(StopWords):\n","    positive_wrds=[]\n","    with open(r\"/content/drive/MyDrive/Text_Analysis/MasterDictionary/positive-words.txt\",\"r\", encoding=\"latin-1\") as f:\n","        positive_wrds+=f.read().split('\\n')\n","    positive_wrds=[wrd for wrd in positive_wrds if wrd not in StopWords]\n","    return positive_wrds"]},{"cell_type":"code","execution_count":8,"metadata":{"id":"EDxXL56ySTRq","executionInfo":{"status":"ok","timestamp":1717056471892,"user_tz":-330,"elapsed":12,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["def negative_dict(StopWords):\n","    negative_wrds=[]\n","    with open(\"/content/drive/MyDrive/Text_Analysis/MasterDictionary/negative-words.txt\",\"r\", encoding=\"latin-1\") as f:\n","        negative_wrds+=f.read().split('\\n')\n","    negative_wrds=[wrd for wrd in negative_wrds if wrd not in StopWords]\n","    return negative_wrds"]},{"cell_type":"markdown","metadata":{"id":"14kpt8UCSTRr"},"source":["# Score"]},{"cell_type":"code","execution_count":9,"metadata":{"id":"Hzmz-urwSTRs","executionInfo":{"status":"ok","timestamp":1717056471892,"user_tz":-330,"elapsed":11,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[],"source":["def sents(text):\n","  lst_of_sent=[(''.join(char for char in sentence if char not in punctuation)) for sentence in nltk.sent_tokenize(text)]\n","  lst_of_sent=[sen for sen in lst_of_sent if sen!=\"\"]\n","  return lst_of_sent\n","\n","def word_count(text):\n","  sentences=sents(text)\n","  stop_words = set(stopwords.words(\"english\"))\n","  wrd_cnt=0\n","  for sen in sentences:\n","    wrd_cnt+=len([word.lower() for word in word_tokenize(sen) if word.isalpha() and word.lower() not in stop_words])\n","  return wrd_cnt\n","\n","def positive_score(text,positive_wrds):\n","  lst_of_wrds=[word.lower() for word in word_tokenize(text) if ((word not in punctuation) and (word in positive_wrds))]\n","  pos_score=len(lst_of_wrds)\n","  return pos_score\n","\n","def negative_score(text,negative_wrds):\n","  lst_of_wrds=[word.lower() for word in word_tokenize(text) if ((word not in punctuation) and (word in negative_wrds))]\n","  neg_score=len(lst_of_wrds)\n","  return neg_score\n","\n","def polarity_score(text,positive_wrds,negative_wrds):\n","  pos=positive_score(text,positive_wrds)\n","  neg=negative_score(text,negative_wrds)\n","  pol_score=(pos-neg)/((pos+neg)+0.000001)\n","  return round(pol_score,2)\n","\n","def subjectivity_score(text,positive_wrds,negative_wrds):\n","  pos=positive_score(text,positive_wrds)\n","  neg=negative_score(text,negative_wrds)\n","  subj_score=(pos+neg)/((word_count(text)+0.000001))\n","  return round(subj_score,2)\n","\n","def avg_sentence_length(text):\n","    no_words = len([word.lower() for word in word_tokenize(text) if (word not in punctuation)])\n","    no_sentences = len(sents(text))\n","    average_sentence_length = round(no_words/no_sentences,2)\n","    return average_sentence_length\n","\n","def avg_word_length(text):\n","  total_words = len([word.lower() for word in word_tokenize(text) if (word not in punctuation)])\n","  total_no_of_char=textstat.letter_count(text, ignore_spaces=True)\n","  avg_word_length=total_no_of_char/total_words\n","  return round(avg_word_length,2)\n","\n","def avg_no_of_words_per_sentences(text):\n","  total_words=len([word.lower() for word in word_tokenize(text) if (word not in punctuation)])\n","  total_sents=len(sents(text))\n","  avg_words_per_sentences=total_words/total_sents\n","  return round(avg_words_per_sentences,2)\n","\n","def fog_index(text):\n","  return round(textstat.gunning_fog(text), 2)\n","\n","def syllable_count(word):\n","    vow=['a','e','i','o','u']\n","    if word.endswith(\"es\") or word.endswith(\"ed\"):\n","        return 0\n","    else:\n","        cnt=0\n","        for chr in word:\n","            if chr in vow:\n","                cnt+=1\n","        return cnt\n","\n","def syllable_cnt_per_word(text):\n","    lst_of_wrds=[word.lower() for word in word_tokenize(text) if (word not in punctuation)]\n","    cnt_syllable_per_wrd=0\n","    for word in lst_of_wrds:\n","        cnt_syllable_per_wrd+=syllable_count(word)\n","    return cnt_syllable_per_wrd\n","\n","def complex_words_count(text):\n","    stop_words = set(stopwords.words(\"english\"))\n","    lst_of_wrds=[word.lower() for word in word_tokenize(text) if (word not in punctuation) and (word not in stop_words)]\n","    comp_words_set = set()\n","    for wrd in lst_of_wrds:\n","        syllable_cnt = syllable_count(wrd)\n","        if syllable_cnt > 2:\n","            comp_words_set.add(wrd)\n","    return len(comp_words_set)\n","\n","def pct_of_complex_wrds(text):\n","  no_complex_words=complex_words_count(text)\n","  no_words=word_count(text)\n","  pct=no_complex_words/no_words\n","  return round(pct,2)\n","\n","def personal_pronouns(text):\n","    pronouns = [\"I\", \"we\", \"my\", \"ours\", \"us\"]\n","    pattern = r'\\b(?:' + '|'.join(pronouns) + r')\\b'\n","    text = re.sub(r'\\b(?:US)\\b', '', text)\n","    matches = re.findall(pattern, text, flags=re.IGNORECASE)\n","    count = len(matches)\n","    return count"]},{"cell_type":"markdown","metadata":{"id":"8FPDYPBpSTRs"},"source":["# Integration"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qvN7PVGLaGvC","outputId":"7a96317d-a3a3-4c05-ec30-09559a61043a","executionInfo":{"status":"ok","timestamp":1717056700301,"user_tz":-330,"elapsed":228419,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Program executed! Please check the Output excel file in the directory.\n"]}],"source":["data = data_extraction()\n","StopWords = stop_words()\n","pos_dic = positive_dict(StopWords)\n","neg_dic = negative_dict(StopWords)\n","\n","df = pd.DataFrame(index=data.index, columns=[\n","    'POSITIVE SCORE',\n","    'NEGATIVE SCORE',\n","    'POLARITY SCORE',\n","    'SUBJECTIVITY SCORE',\n","    'AVG SENTENCE LENGTH',\n","    'PERCENTAGE OF COMPLEX WORDS',\n","    'FOG INDEX',\n","    'AVG NUMBER OF WORDS PER SENTENCE',\n","    'COMPLEX WORD COUNT',\n","    'WORD COUNT',\n","    'SENTENCE COUNT',\n","    'SYLLABLE PER WORD',\n","    'PERSONAL PRONOUNS',\n","    'AVG WORD LENGTH'\n","])\n","\n","for i in range(len(data)):\n","    txt = data.iloc[i].text\n","    SentCount = len(sents(txt))\n","    WordCount = word_count(txt)\n","    PositiveScore = positive_score(txt, pos_dic)\n","    NegativeScore = negative_score(txt, neg_dic)\n","    PolarityScore = polarity_score(txt, pos_dic, neg_dic)\n","    SubjectiveScore = subjectivity_score(txt, pos_dic, neg_dic)\n","    AvgSentenceLen = avg_sentence_length(txt)\n","    AvgWordLen = avg_word_length(txt)\n","    AvgWordsPerSentences = avg_no_of_words_per_sentences(txt)\n","    FogIndx = fog_index(txt)\n","    SyllableCntPerWord = syllable_cnt_per_word(txt)\n","    ComplxWordCount = complex_words_count(txt)\n","    PercentageComplexWord = pct_of_complex_wrds(txt)\n","    PersonalPronoun = personal_pronouns(txt)\n","\n","    df.iloc[i] = [PositiveScore, NegativeScore, PolarityScore, SubjectiveScore, AvgSentenceLen, PercentageComplexWord,\n","                  FogIndx, AvgWordsPerSentences, ComplxWordCount, WordCount, SentCount, SyllableCntPerWord, PersonalPronoun,\n","                  AvgWordLen]\n","\n","final_df = pd.concat([data, df], axis=1)\n","final_df.drop(['title', 'text'], axis=1, inplace=True)\n","final_df.to_excel(r\"/content/drive/MyDrive/Text_Analysis/Output.xlsx\")\n","\n","print(\"Program executed! Please check the Output excel file in the directory.\")\n"]},{"cell_type":"code","source":[],"metadata":{"id":"fX6eSwurx2WF","executionInfo":{"status":"ok","timestamp":1717056700301,"user_tz":-330,"elapsed":7,"user":{"displayName":"Eakta Prasad","userId":"07869702588292411068"}}},"execution_count":10,"outputs":[]}],"metadata":{"colab":{"provenance":[],"gpuType":"T4"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.9"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":0}