Data_Extraction_and_Text_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Data_Extraction_and_Text_Analysis.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Manojpatil123/Data-extraction-and-text-analysis/blob/main/Data_Extraction_and_Text_Analysis.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qc6rCk68pjiI"
      },
      "outputs": [],
      "source": [
        "#importing library\n",
        "import pandas as pd\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import numpy as np\n",
        "from google.colab import files\n",
        "import nltk\n",
        "import string\n",
        "nltk.download('punkt')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EZOpO9Wr2N-q",
        "outputId": "a712f270-5db9-471e-b1a9-61cd327f38e3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#importing input file\n",
        "df=pd.read_csv('/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/Input.csv')[['URL_ID','URL']]"
      ],
      "metadata": {
        "id": "j3wAqjmppkVC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "df=df.iloc[0:150]"
      ],
      "metadata": {
        "id": "SrEPOMu2pkPk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "kqvyVaiF3XAt",
        "outputId": "fcf2ad2c-88e7-4144-96a8-ae5e86854b5d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     URL_ID                                                URL\n",
              "0       1.0  https://insights.blackcoffer.com/is-telehealth...\n",
              "1       2.0  https://insights.blackcoffer.com/how-telehealt...\n",
              "2       3.0  https://insights.blackcoffer.com/is-telemedici...\n",
              "3       4.0  https://insights.blackcoffer.com/is-telehealth...\n",
              "4       5.0  https://insights.blackcoffer.com/how-people-di...\n",
              "..      ...                                                ...\n",
              "145   146.0  https://insights.blackcoffer.com/blockchain-fo...\n",
              "146   147.0  https://insights.blackcoffer.com/the-future-of...\n",
              "147   148.0  https://insights.blackcoffer.com/big-data-anal...\n",
              "148   149.0  https://insights.blackcoffer.com/business-anal...\n",
              "149   150.0  https://insights.blackcoffer.com/challenges-an...\n",
              "\n",
              "[150 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-b849c7bd-8b36-4d71-96b2-f91389ccfdf9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>URL_ID</th>\n",
              "      <th>URL</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1.0</td>\n",
              "      <td>https://insights.blackcoffer.com/is-telehealth...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2.0</td>\n",
              "      <td>https://insights.blackcoffer.com/how-telehealt...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3.0</td>\n",
              "      <td>https://insights.blackcoffer.com/is-telemedici...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4.0</td>\n",
              "      <td>https://insights.blackcoffer.com/is-telehealth...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5.0</td>\n",
              "      <td>https://insights.blackcoffer.com/how-people-di...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>145</th>\n",
              "      <td>146.0</td>\n",
              "      <td>https://insights.blackcoffer.com/blockchain-fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>146</th>\n",
              "      <td>147.0</td>\n",
              "      <td>https://insights.blackcoffer.com/the-future-of...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>148.0</td>\n",
              "      <td>https://insights.blackcoffer.com/big-data-anal...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148</th>\n",
              "      <td>149.0</td>\n",
              "      <td>https://insights.blackcoffer.com/business-anal...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>149</th>\n",
              "      <td>150.0</td>\n",
              "      <td>https://insights.blackcoffer.com/challenges-an...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>150 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b849c7bd-8b36-4d71-96b2-f91389ccfdf9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-b849c7bd-8b36-4d71-96b2-f91389ccfdf9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-b849c7bd-8b36-4d71-96b2-f91389ccfdf9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.drop('URL_ID',axis=1,inplace=True)"
      ],
      "metadata": {
        "id": "m_-j2x4j3W-G"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data extraction"
      ],
      "metadata": {
        "id": "G0qbaPKada0c"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#extracting text from all the url\n",
        "url_id=1\n",
        "for i in range(0,len(df)):\n",
        " \n",
        "   j=df.iloc[i].values\n",
        "   \n",
        "   headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'}#giving user access\n",
        "   page=requests.get(j[0],headers=headers)#loading text in url\n",
        "   soup=BeautifulSoup(page.content,'html.parser')#parsing url text\n",
        "   content=soup.findAll(attrs={'class':'td-post-content'})#extracting only text part\n",
        "   content=content[0].text.replace('\\xa0',\"  \").replace('\\n',\"  \")#replace end line symbol with space \n",
        "   title=soup.findAll(attrs={'class':'entry-title'})#extracting title of website\n",
        "   title=title[16].text.replace('\\n',\"  \").replace('/',\"\")\n",
        "   text=title+ '.' +content#merging title and content text\n",
        "   text=np.array(text)#converting to array form\n",
        "   text.reshape(1,-1)#changing shape to 1d \n",
        "   df1=pd.Series(text)#creating series data frame\n",
        "   b=str(url_id)+\".\"+'txt'#name of the text file\n",
        "  # with open(b, 'a') as f:#creating text file \n",
        "   # df1.to_csv(f, line_terminator=',', index=False, header=False)\n",
        "  # files.download(b)#downloading text file\n",
        "   url_id+=1\n",
        "   "
      ],
      "metadata": {
        "id": "czsKjlCH3W32"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Data analysis"
      ],
      "metadata": {
        "id": "ETT5Ce93dW3E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#importing each extracted files\n",
        "text=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/data extract to text/150.txt\",header=None)"
      ],
      "metadata": {
        "id": "gHvRnhT3c3pc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#information of data frame\n",
        "text.info()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "99yHj18o_iDU",
        "outputId": "9bd6dc9b-95b8-4963-f15c-230c2cb38760"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 1 entries, 0 to 0\n",
            "Data columns (total 2 columns):\n",
            " #   Column  Non-Null Count  Dtype  \n",
            "---  ------  --------------  -----  \n",
            " 0   0       1 non-null      object \n",
            " 1   1       0 non-null      float64\n",
            "dtypes: float64(1), object(1)\n",
            "memory usage: 144.0+ bytes\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#removing extra created column\n",
        "text.drop(1,axis=1,inplace=True)"
      ],
      "metadata": {
        "id": "PSk_xA-XAoQz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#converting type\n",
        "text=text.astype(str)"
      ],
      "metadata": {
        "id": "dzIKncs5BJEj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#converting text to sentence\n",
        "import re\n",
        "a=text[0].str.split('([\\.]\\s)',expand=False)#splitting text on '.'\n",
        "b=a.explode()#converting to rows\n",
        "b=pd.DataFrame(b)#creating data frame\n",
        "b.columns=['abc']\n"
      ],
      "metadata": {
        "id": "KCDJzEnU7Ogk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#removing . char from each rows\n",
        "def abcd(x):    \n",
        "    nopunc =[char for char in x if char != '.']\n",
        "    return ''.join(nopunc)\n",
        "b['abc']=b['abc'].apply(abcd)\n"
      ],
      "metadata": {
        "id": "-V-qW6SbTQMW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#replacing emty space with null values\n",
        "c=b.replace('',np.nan,regex=True)\n",
        "c=c.mask(c==\" \")\n",
        "c=c.dropna()\n",
        "c.reset_index(drop=True,inplace=True)"
      ],
      "metadata": {
        "id": "NYwn1576looc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "c"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "yikIBkKzeoBM",
        "outputId": "7485cd91-745e-4540-fbdb-77d3936541c3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  abc\n",
              "0   Challenges and Opportunities of Big Data in He...\n",
              "1    Big Data  To begin with I shall first like to...\n",
              "2       Big data is simply data, but with a huge size\n",
              "3   Data that is not just voluminous but also grow...\n",
              "4   Such data sets are so large and complex that i...\n",
              "..                                                ...\n",
              "61  Scarce resources like  data scientists are har...\n",
              "62            They are easily  poached by competitors\n",
              "63  A good and efficient compensation strategy, co...\n",
              "64    In a nutshell, we can conclude that while bi...\n",
              "65   Blackcoffer Insights 10 | Subhasmita Dey, Xav...\n",
              "\n",
              "[66 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e36302c0-31ff-4241-a9bb-1cbcde591f48\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>abc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Challenges and Opportunities of Big Data in He...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Big Data  To begin with I shall first like to...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Big data is simply data, but with a huge size</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Data that is not just voluminous but also grow...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Such data sets are so large and complex that i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>Scarce resources like  data scientists are har...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>They are easily  poached by competitors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>A good and efficient compensation strategy, co...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>In a nutshell, we can conclude that while bi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>Blackcoffer Insights 10 | Subhasmita Dey, Xav...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>66 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e36302c0-31ff-4241-a9bb-1cbcde591f48')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e36302c0-31ff-4241-a9bb-1cbcde591f48 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e36302c0-31ff-4241-a9bb-1cbcde591f48');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8032
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#importing nltk library and stopwords\n",
        "import nltk\n",
        "import string\n"
      ],
      "metadata": {
        "id": "iyBSH3bJc3ms"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "punc=[punc for punc in string.punctuation]"
      ],
      "metadata": {
        "id": "WLtSU4mXm4Rp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "punc"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ht4ZLTmlQhp9",
        "outputId": "a6ee7d09-6bef-4243-d831-42e48119e8cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['!',\n",
              " '\"',\n",
              " '#',\n",
              " '$',\n",
              " '%',\n",
              " '&',\n",
              " \"'\",\n",
              " '(',\n",
              " ')',\n",
              " '*',\n",
              " '+',\n",
              " ',',\n",
              " '-',\n",
              " '.',\n",
              " '/',\n",
              " ':',\n",
              " ';',\n",
              " '<',\n",
              " '=',\n",
              " '>',\n",
              " '?',\n",
              " '@',\n",
              " '[',\n",
              " '\\\\',\n",
              " ']',\n",
              " '^',\n",
              " '_',\n",
              " '`',\n",
              " '{',\n",
              " '|',\n",
              " '}',\n",
              " '~']"
            ]
          },
          "metadata": {},
          "execution_count": 8035
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#importing stop words files that are provided\n",
        "StopWords_Auditor=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_Auditor.txt\",header=None)\n",
        "StopWords_Currencies=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_Currencies.txt\",header=None,encoding=\"ISO-8859-1\",sep='\\n' )\n",
        "StopWords_DatesandNumbers=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_DatesandNumbers.txt\",header=None)\n",
        "StopWords_Generic=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_Generic.txt\",header=None)\n",
        "StopWords_GenericLong=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_GenericLong.txt\",header=None)\n",
        "StopWords_Geographic=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_Geographic.txt\",header=None)\n",
        "StopWords_Names=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/StopWords-20220705T045434Z-001/StopWords/StopWords_Names.txt\",header=None)\n"
      ],
      "metadata": {
        "id": "s1xGY76Fm4OB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#creating func for removing stop words\n",
        "def text_process(text):\n",
        "    nopunc =[char for char in text if char not in punc or char not in [':',',','(',')','’','?']]\n",
        "    nopunc=''.join(nopunc)\n",
        "    txt=' '.join([word for word in nopunc.split() if word.lower() not in StopWords_Auditor])\n",
        "    txt1=' '.join([word for word in txt.split() if word.lower() not in StopWords_Currencies])\n",
        "    txt2=' '.join([word for word in txt1.split() if word.lower() not in StopWords_DatesandNumbers])\n",
        "    txt3=' '.join([word for word in txt2.split() if word.lower() not in StopWords_Generic])\n",
        "    txt4=' '.join([word for word in txt3.split() if word.lower() not in StopWords_GenericLong])\n",
        "    txt5=' '.join([word for word in txt4.split() if word.lower() not in StopWords_Geographic])\n",
        "    return ' '.join([word for word in txt5.split() if word.lower() not in StopWords_Names])\n"
      ],
      "metadata": {
        "id": "sX1qAALqm4Kz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#applying func for each row\n",
        "c['abc']=c['abc'].apply(text_process)"
      ],
      "metadata": {
        "id": "RH0fzgVym4H6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "c"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "0Nv3eVe7IahQ",
        "outputId": "1e8e3b07-db5e-4a5c-8dd7-a43dc1ac77d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  abc\n",
              "0   Challenges and Opportunities of Big Data in He...\n",
              "1   Big Data To begin with I shall first like to e...\n",
              "2        Big data is simply data but with a huge size\n",
              "3   Data that is not just voluminous but also grow...\n",
              "4   Such data sets are so large and complex that i...\n",
              "..                                                ...\n",
              "61  Scarce resources like data scientists are hard...\n",
              "62             They are easily poached by competitors\n",
              "63  A good and efficient compensation strategy con...\n",
              "64  In a nutshell we can conclude that while big d...\n",
              "65  Blackcoffer Insights 10 | Subhasmita Dey Xavie...\n",
              "\n",
              "[66 rows x 1 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c34e1aec-e561-4df0-8680-59abc77229f3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>abc</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Challenges and Opportunities of Big Data in He...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Big Data To begin with I shall first like to e...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Big data is simply data but with a huge size</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Data that is not just voluminous but also grow...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Such data sets are so large and complex that i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>61</th>\n",
              "      <td>Scarce resources like data scientists are hard...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>62</th>\n",
              "      <td>They are easily poached by competitors</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>63</th>\n",
              "      <td>A good and efficient compensation strategy con...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>64</th>\n",
              "      <td>In a nutshell we can conclude that while big d...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>65</th>\n",
              "      <td>Blackcoffer Insights 10 | Subhasmita Dey Xavie...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>66 rows × 1 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c34e1aec-e561-4df0-8680-59abc77229f3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c34e1aec-e561-4df0-8680-59abc77229f3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c34e1aec-e561-4df0-8680-59abc77229f3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8039
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#importing master Dictionary\n",
        "positive=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/MasterDictionary/positive-words.txt\",header=None)\n",
        "negative=pd.read_csv(\"/content/drive/MyDrive/Test assignment/Blackcoffer Data Science internship/MasterDictionary/negative-words.txt\",header=None,encoding=\"ISO-8859-1\",sep='\\n' )\n"
      ],
      "metadata": {
        "id": "s-ccXyf4IaeY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "positive.columns=['abc']\n",
        "negative.columns=['abc']\n",
        "positive['abc']=positive['abc'].astype(str)\n",
        "negative['abc']=negative['abc'].astype(str)"
      ],
      "metadata": {
        "id": "jKpEw9KuH8vh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#positive and negative dictionary without stopwords\n",
        "positive['abc']=positive['abc'].apply(text_process)\n",
        "negative['abc']=negative['abc'].apply(text_process)"
      ],
      "metadata": {
        "id": "Cit_gYSDSm6p"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#positive list\n",
        "length=positive.shape[0]\n",
        "post=[]\n",
        "for i in range(0,length):\n",
        "   nopunc =[char for char in positive.iloc[i] if char not in string.punctuation or char != '+']\n",
        "   nopunc=''.join(nopunc)\n",
        "\n",
        "   post.append(nopunc)"
      ],
      "metadata": {
        "id": "9BZR-CHpVIwx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#negative list\n",
        "length=negative.shape[0]\n",
        "neg=[]\n",
        "for i in range(0,length):\n",
        "  nopunc =[char for char in negative.iloc[i] if char not in string.punctuation or char != '+']\n",
        "  nopunc=''.join(nopunc)\n",
        "  neg.append(nopunc)"
      ],
      "metadata": {
        "id": "RDz17VIAPp8Q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#importing tokenize library\n",
        "from nltk.tokenize import word_tokenize"
      ],
      "metadata": {
        "id": "sQ4IRcm1RseT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "txt_list=[]\n",
        "length=c.shape[0]\n",
        "for i in range(0,length):\n",
        "  txt=' '.join([word for word in c.iloc[i]])\n",
        "  txt_list.append(txt)"
      ],
      "metadata": {
        "id": "ovZ18CdBN2-i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#tokenization of text\n",
        "tokenize_text=[]\n",
        "for i in txt_list:\n",
        "  \n",
        "  tokenize_text+=(word_tokenize(i))"
      ],
      "metadata": {
        "id": "9Uk7eZZwSaYh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(tokenize_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0nAoWCr5fsZz",
        "outputId": "a0564ff0-9359-424a-d794-0780942eb29f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Challenges', 'and', 'Opportunities', 'of', 'Big', 'Data', 'in', 'Healthcare', 'Big', 'Data', 'To', 'begin', 'with', 'I', 'shall', 'first', 'like', 'to', 'explain', 'what', 'big', 'data', 'is', 'and', 'why', 'it', 'has', 'become', 'so', 'important', 'in', 'our', 'lives', 'Big', 'data', 'is', 'simply', 'data', 'but', 'with', 'a', 'huge', 'size', 'Data', 'that', 'is', 'not', 'just', 'voluminous', 'but', 'also', 'growing', 'exponentially', 'with', 'time', 'Such', 'data', 'sets', 'are', 'so', 'large', 'and', 'complex', 'that', 'it', 'is', 'not', 'possible', 'to', 'store', 'or', 'process', 'them', 'using', 'traditional', 'data', 'management', 'tools', 'So', 'how', 'huge', 'can', 'this', 'data', 'be', 'Social', 'media', 'sites', 'like', 'Facebook', 'generate', 'more', 'than', '500', 'terabytes', 'of', 'new', 'data', 'every', 'day', 'in', 'the', 'form', 'of', 'photos', 'video', 'uploads', 'text', 'messages', 'etc', 'A', 'single', 'Jet', 'engine', 'can', 'generate', 'more', 'than', '10', 'terabytes', 'of', 'data', 'in', '30', 'minutes', 'of', 'flight', 'time', 'With', 'many', 'thousand', 'flights', 'per', 'day', 'generation', 'of', 'data', 'reaches', 'up', 'to', 'many', 'petabytes', 'Data', 'contained', 'in', 'these', 'sets', 'are', 'not', 'always', 'structured', 'It', 'can', 'be', 'semi-structured', 'or', 'even', 'unstructured', 'When', 'such', 'is', 'the', 'size', 'and', 'dimension', 'of', 'data', 'we', 'can', 'well', 'imagine', 'how', 'complicated', 'it', 'must', 'be', 'to', 'process', 'and', 'analyze', 'it', 'Therefore', 'modern', 'methods', 'of', 'data', 'analysis', 'are', 'being', 'developed', 'and', 'used', 'to', 'process', 'the', 'information', 'contained', 'in', 'big', 'data', 'sets', 'Opportunities', 'Now', 'coming', 'to', 'the', 'opportunities', 'that', 'Big', 'Data', 'provides', 'they', 'are', 'not', 'just', 'limited', 'to', 'healthcare', 'Big', 'data', 'analysis', 'is', 'now', 'a', 'disruptive', 'technology', 'which', 'has', 'intervened', 'into', 'numerous', 'fields', 'and', 'proved', 'its', 'worth', 'Healthcare', 'is', 'no', 'exception', 'Big', 'data', 'has', 'the', 'potential', 'to', 'change', 'the', 'entire', 'dynamics', 'of', 'the', 'healthcare', 'industry', 'and', 'improve', 'the', 'quality', 'of', 'life', 'of', 'people', 'Healthcare', 'industry', 'is', 'a', 'very', 'large', 'and', 'complicated', 'system', 'It', 'involves', 'a', 'lot', 'of', 'risks', 'and', 'always', 'demands', 'better', 'care', 'However', 'when', 'a', 'large', 'number', 'of', 'patients', 'seek', 'emergency', 'care', 'the', 'complications', 'as', 'well', 'as', 'the', 'cost', 'rise', 'exponentially', 'In', 'India', 'many', 'times', 'we', 'even', 'lack', 'sufficient', 'infrastructure', 'to', 'support', 'the', 'patients', 'there', 'is', 'a', 'deficit', 'of', 'beds', 'as', 'well', 'as', 'doctors', 'to', 'provide', 'treatment', 'to', 'the', 'patients', 'For', 'instance', 'when', 'epidemics', 'break', 'out', 'and', 'lives', 'are', 'lost', 'at', 'a', 'very', 'alarming', 'rate', 'we', 'can', 'easily', 'gauge', 'our', 'helplessness', 'However', 'the', 'scenario', 'is', 'certainly', 'improving', 'now', 'With', 'the', 'advent', 'of', 'digitization', 'into', 'the', 'healthcare', 'system', 'healthcare', 'providers', 'or', 'practitioners', 'are', 'now', 'having', 'access', 'to', 'a', 'huge', 'amount', 'of', 'patient', 'health', 'data', 'This', 'healthcare', 'big', 'data', 'can', 'be', 'processed', 'and', 'analyzed', 'to', 'identify', 'patient', 'patterns', 'more', 'quickly', 'and', 'effectively', 'The', 'information', 'obtained', 'can', 'be', 'extremely', 'useful', 'to', 'figure', 'out', 'chronic', 'health', 'issues', 'and', 'provide', 'preventive', 'treatment', 'plans', 'well', 'beforehand', 'so', 'as', 'to', 'curb', 'that', 'disease', 'or', 'disorder', 'from', 'occurring', 'This', 'method', 'is', 'also', 'known', 'as', 'predictive', 'analysis', 'and', 'it', 'is', 'one', 'of', 'the', 'most', 'crucial', 'benefits', 'of', 'big', 'data', 'in', 'healthcare', 'This', 'technology', 'can', 'help', 'the', 'healthcare', 'industry', 'in', 'more', 'ways', 'than', 'we', 'can', 'infer', 'Healthcare', 'organizations', 'that', 'have', 'implemented', 'predictive', 'analysis', 'have', 'witnessed', 'a', 'reduction', 'in', 'ER', 'visits', 'by', 'providing', 'support', 'and', 'care', 'to', 'patients', 'and', 'decreasing', 'emergency', 'situations', 'Apart', 'from', 'reduced', 'ER', 'visits', 'and', 'timely', 'treatment', 'there', 'are', 'many', 'other', 'benefits', 'of', 'big', 'data', 'and', 'predictive', 'analysis', 'Patients', 'with', 'high-risk', 'life-threatening', 'issues', 'can', 'be', 'provided', 'with', 'more', 'customized', 'treatment', 'facilities', 'Due', 'to', 'lack', 'of', 'data', 'there', 'exists', 'a', 'lack', 'of', 'proper', 'planning', 'in', 'hospitals', 'Under', 'or', 'over-booking', 'of', 'staff', 'lack', 'of', 'medical', 'equipment', 'medicines', 'and', 'other', 'facilities', 'are', 'a', 'result', 'of', 'inefficient', 'budgeting', 'and', 'ill-management', 'of', 'finances', 'Using', 'predictive', 'analysis', 'these', 'problems', 'can', 'be', 'solved', 'which', 'will', 'lead', 'to', 'reduced', 'costs', 'and', 'efficient', 'management', 'of', 'finances', 'Better', 'staff', 'allocation', 'and', 'admission', 'rate', 'prediction', 'shall', 'facilitate', 'improvement', 'in', 'daily', 'operations', 'of', 'healthcare', 'organizations', 'By', 'using', 'big', 'data', 'the', 'effect', 'of', 'recency', 'bias', 'could', 'be', 'reduced', 'as', 'well', 'When', 'we', 'give', 'more', 'importance', 'to', 'recent', 'events', 'and', 'tend', 'to', 'ignore', 'the', 'effect', 'of', 'the', 'older', 'ones', 'it', 'may', 'lead', 'to', 'incorrect', 'decisions', 'This', 'is', 'known', 'as', 'recency', 'bias', 'Big', 'data', 'also', 'helps', 'in', 'preventing', 'fraudulent', 'activities', 'which', 'in', 'turn', 'prevents', 'losses', 'of', 'insurance', 'companies', 'Challenges', 'While', 'all', 'of', 'this', 'is', 'changing', 'the', 'healthcare', 'industry', 'for', 'the', 'better', 'it', 'is', 'not', 'that', 'easy', 'to', 'reap', 'the', 'benefits', 'of', 'big', 'data', 'There', 'are', 'a', 'whole', 'lot', 'of', 'challenges', 'and', 'vulnerabilities', 'attached', 'to', 'its', 'implementation', 'One', 'of', 'the', 'biggest', 'challenges', 'is', 'security', 'Healthcare', 'big', 'data', 'contains', 'the', 'personal', 'information', 'and', 'health', 'history', 'of', 'patients', 'Acts', 'of', 'hacking', 'cyber', 'theft', 'and', 'phishing', 'pose', 'a', 'serious', 'threat', 'to', 'these', 'databases', 'Such', 'data', 'could', 'be', 'stolen', 'and', 'sold', 'for', 'huge', 'sums', 'of', 'money', 'Protection', 'of', 'the', 'patients', '’', 'privacy', 'hence', 'is', 'a', 'serious', 'challenge', 'to', 'big', 'data', 'implementation', 'Also', 'the', 'data', 'would', 'contain', 'external', 'data', 'apart', 'from', 'medical', 'information', 'The', 'organization', 'therefore', 'has', 'to', 'take', 'care', 'of', 'privacy', 'legal', 'compliances', 'and', 'government', 'policies', 'Privacy', 'and', 'security', 'of', 'patients', 'have', 'to', 'be', 'given', 'utmost', 'importance', 'and', 'no', 'breach', 'of', 'any', 'kind', 'can', 'be', 'permitted', 'The', 'next', 'challenge', 'is', 'data', 'classification', 'and', 'modeling', 'The', 'size', 'of', 'the', 'data', 'is', 'massive', 'and', 'it', 'is', 'less', 'structured', 'and', 'heterogeneous', 'Classifying', 'such', 'massive', 'data', 'to', 'identify', 'relevant', 'information', 'is', 'a', 'big', 'challenge', 'Modeling', 'of', 'such', 'unstructured', 'data', 'is', 'equally', 'difficult', 'Storage', 'and', 'retrieval', 'is', 'another', 'major', 'challenge', 'Huge', 'cloud', 'servers', 'with', 'sufficient', 'space', 'are', 'required', 'to', 'store', 'such', 'voluminous', 'data', 'Also', 'the', 'speed', 'should', 'be', 'high', 'so', 'that', 'uploading', 'of', 'data', 'can', 'be', 'done', 'hassle-free', 'The', 'way', 'storage', 'is', 'a', 'challenge', 'retrieval', 'also', 'is', 'a', 'matter', 'of', 'concern', 'Integrating', 'the', 'data', 'and', 'getting', 'all', 'relevant', 'systems', 'to', 'link', 'each', 'other', 'is', 'a', 'tough', 'job', 'The', 'next', 'challenge', 'is', 'a', 'major', 'one', 'in', 'my', 'opinion', 'Finding', 'the', 'right', 'talent', 'who', 'own', 'the', 'expertise', 'to', 'implement', 'this', 'modern', 'technology', 'is', 'an', 'arduous', 'task', 'Shortage', 'of', 'required', 'talent', 'is', 'a', 'crisis', 'in', 'the', 'market', 'today', 'Even', 'after', 'hiring', 'the', 'right', 'talent', 'it', 'is', 'a', 'challenge', 'to', 'retain', 'them', 'Scarce', 'resources', 'like', 'data', 'scientists', 'are', 'hard', 'to', 'find', 'and', 'even', 'harder', 'to', 'retain', 'They', 'are', 'easily', 'poached', 'by', 'competitors', 'A', 'good', 'and', 'efficient', 'compensation', 'strategy', 'conducive', 'work', 'environment', 'high', 'incentives', 'opportunities', 'for', 'career', 'growth', 'and', 'development', 'can', 'be', 'some', 'of', 'the', 'ways', 'of', 'retaining', 'such', 'intellectual', 'talent', 'In', 'a', 'nutshell', 'we', 'can', 'conclude', 'that', 'while', 'big', 'data', 'is', 'a', 'disruptive', 'technology', 'which', 'will', 'bring', 'about', 'landmark', 'changes', 'in', 'healthcare', 'dynamics', 'the', 'challenges', 'and', 'vulnerabilities', 'need', 'to', 'be', 'addressed', 'with', 'the', 'utmost', 'care', 'and', 'sense', 'of', 'responsibility', 'Blackcoffer', 'Insights', '10', '|', 'Subhasmita', 'Dey', 'Xavier', 'School', 'of', 'Human', 'Resource', 'Management', 'XUB']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(tokenize_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2U5HOJqTT8j7",
        "outputId": "85866329-a6db-42ca-c0ed-c45760213850"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1055"
            ]
          },
          "metadata": {},
          "execution_count": 8049
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1) POSITIVE SCORE"
      ],
      "metadata": {
        "id": "LEV13sFhOtdZ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "positive_score=0\n",
        "for i in tokenize_text:\n",
        "  if(i.lower() in post):\n",
        "    positive_score+=1\n",
        "print('postive score=', positive_score)"
      ],
      "metadata": {
        "id": "PFskHGo4hH-D",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "df9f26fe-effb-4e8d-ad34-55e5ed745049"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "postive score= 47\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 2) NEGATIVE SCORE"
      ],
      "metadata": {
        "id": "TbnTPNmaPFZY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "negative_score=0\n",
        "for i in tokenize_text:\n",
        "  if(i.lower() in neg):\n",
        "    negative_score+=1\n",
        "print('negative score=', negative_score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NW-BmJrrLITJ",
        "outputId": "ced4aa47-d58f-4944-a572-8d053ea66899"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "negative score= 41\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 3) POLARITY SCORE"
      ],
      "metadata": {
        "id": "nOzdsFrmPs0Q"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Polarity Score = (Positive Score – Negative Score)/ ((Positive Score + Negative Score) + 0.000001)\n",
        "Polarity_Score=(positive_score-negative_score)/((positive_score+negative_score)+0.000001)\n",
        "print('polarity_score=', Polarity_Score)"
      ],
      "metadata": {
        "id": "9dpYkSQ2hH3s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5d82536f-c528-4964-9f7f-fc7c988c9a11"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "polarity_score= 0.06818181740702481\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4) SUBJECTIVITY SCORE"
      ],
      "metadata": {
        "id": "ROhnqKrnSLtv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Subjectivity Score = (Positive Score + Negative Score)/ ((Total Words after cleaning) + 0.000001)\n",
        "subjectiivity_score=(positive_score-negative_score)/((len(tokenize_text))+ 0.000001)\n",
        "print('subjectivity_score',subjectiivity_score)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FWLTdqkgSMBM",
        "outputId": "279e7b09-a37b-4bcf-e747-f65fbb1469ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "subjectivity_score 0.005687203786078479\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 5) AVG SENTENCE LENGTH"
      ],
      "metadata": {
        "id": "9TQxzV5EU1rz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "length=c.shape[0]\n",
        "avg_length=[]\n",
        "for i in range(0,length):\n",
        "  avg_length.append(len(c['abc'].iloc[i]))\n",
        "avg_senetence_length=sum(avg_length)/len(avg_length)\n",
        "print('avg sentence length=', avg_senetence_length)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2yKBk8AzULAh",
        "outputId": "60f088ca-82cd-4eba-ff6c-48b7140a4d2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "avg sentence length= 95.54545454545455\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 6) PERCENTAGE OF COMPLEX WORDS"
      ],
      "metadata": {
        "id": "90O5o3FHa0Ql"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vowels=['a','e','i','o','u']\n",
        "import re\n",
        "count=0\n",
        "complex_Word_Count=0\n",
        "for i in tokenize_text:\n",
        "  x=re.compile('[es|ed]$')\n",
        "  if x.match(i.lower()):\n",
        "   count+=0\n",
        "  else:\n",
        "    for j in i:\n",
        "      if(j.lower() in vowels ):\n",
        "        count+=1\n",
        "  if(count>2):\n",
        "   complex_Word_Count+=1\n",
        "  count=0\n"
      ],
      "metadata": {
        "id": "9Wx95gemUK9X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "Percentage_of_Complex_words=complex_Word_Count/len(tokenize_text)\n",
        "print('percentag of complex words= ',Percentage_of_Complex_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4KfZE9shUK6H",
        "outputId": "f5a5ddd2-5729-48d5-cc66-b3b6219e7219"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "percentag of complex words=  0.26445497630331755\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 7) FOG INDEX"
      ],
      "metadata": {
        "id": "wSn_zCTlgMTF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Fog Index = 0.4 * (Average Sentence Length + Percentage of Complex words)\n",
        "Fog_Index = 0.4 * (avg_senetence_length + Percentage_of_Complex_words)\n",
        "print('fog index= ',Fog_Index )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xd_zHORCflz8",
        "outputId": "ee73b4b8-0a5e-49bb-94fd-4eb0d1438b4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "fog index=  38.32396380870315\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 8) AVG NUMBER OF WORDS PER SENTENCE"
      ],
      "metadata": {
        "id": "z6tbgChOg0MT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "length=c.shape[0]\n",
        "avg_length=[]\n",
        "for i in range(0,length):\n",
        "  a=[word.split( ) for word in c.iloc[i]]\n",
        "  avg_length.append(len(a[0]))\n",
        "  a=0\n",
        "#avg\n",
        "avg_no_of_words_per_sentence=sum(avg_length)/length\n",
        "print(\"avg no of words per sentence= \",avg_no_of_words_per_sentence)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2NjRGOeWh3ih",
        "outputId": "6ce60e16-8353-418d-cd5c-7d65d581071c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "avg no of words per sentence=  15.969696969696969\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 9) COMPLEX WORD COUNT"
      ],
      "metadata": {
        "id": "0gERUJr_lR4L"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vowels=['a','e','i','o','u']\n",
        "import re\n",
        "count=0\n",
        "complex_Word_Count=0\n",
        "for i in tokenize_text:\n",
        "  x=re.compile('[es|ed]$')\n",
        "  if x.match(i.lower()):\n",
        "   count+=0\n",
        "  else:\n",
        "    for j in i:\n",
        "      if(j.lower() in vowels ):\n",
        "        count+=1\n",
        "  if(count>2):\n",
        "   complex_Word_Count+=1\n",
        "  count=0\n",
        "print('complex words count=',  complex_Word_Count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zuNyk0f0h3gG",
        "outputId": "d10c4b86-5272-4be4-a2eb-74d07276106f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "complex words count= 279\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 10) WORD COUNT"
      ],
      "metadata": {
        "id": "EDbM2keclqrj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "word_count=len(tokenize_text)\n",
        "print('word count= ', word_count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5mwte9rch3c7",
        "outputId": "559fd9a3-c3db-4a8d-c967-cc2a11c6636d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "word count=  1055\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 11) SYLLABLE PER WORD"
      ],
      "metadata": {
        "id": "zfqS5QdcmAdK"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vowels=['a','e','i','o','u']\n",
        "import re\n",
        "count=0\n",
        "for i in tokenize_text:\n",
        "  x=re.compile('[es|ed]$')\n",
        "  if x.match(i.lower()):\n",
        "   count+=0\n",
        "  else:\n",
        "    for j in i:\n",
        "      if(j.lower() in vowels ):\n",
        "        count+=1\n",
        "syllable_count=count\n",
        "print('syllable_per_word= ',syllable_count)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CDgf9sz1h3aL",
        "outputId": "e4409c6b-8c64-47ad-c897-fcc724834ada"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "syllable_per_word=  2065\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 12) PERSONAL PRONOUNS"
      ],
      "metadata": {
        "id": "BcBCr8CtmuoS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pronouns=['i','we','my','ours','us' ]\n",
        "import re\n",
        "count=0\n",
        "for i in tokenize_text:\n",
        "  if i.lower() in pronouns:\n",
        "   count+=1\n",
        "personal_pronouns=count\n",
        "print('personal pronouns= ',personal_pronouns )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aeRcGapymu-D",
        "outputId": "1f791480-f429-473f-959a-7cbaadd7261f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "personal pronouns=  8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 13) AVG WORD LENGTH"
      ],
      "metadata": {
        "id": "-uWlAk9roY5y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "count=0\n",
        "for i in tokenize_text:\n",
        "  for j in i:\n",
        "    count+=1\n",
        "avg_word_length=count/len(tokenize_text)\n",
        "print('avg word= ', avg_word_length)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EZzeKglfn225",
        "outputId": "4db590c0-9a2b-4350-99d4-a7165fe10b4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "avg word=  5.040758293838863\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data={'positive_score':positive_score,'negative_score':negative_score,'Polarity_Score':Polarity_Score,'subjectiivity_score':subjectiivity_score,'avg_senetence_length':avg_senetence_length,'Percentage_of_Complex_words':Percentage_of_Complex_words,'Fog_Index':Fog_Index,'avg_no_of_words_per_sentence':avg_no_of_words_per_sentence,'complex_Word_Count':complex_Word_Count,'word_count':word_count,'syllable_count':syllable_count,'personal_pronouns':personal_pronouns,'avg_word_length':avg_word_length}"
      ],
      "metadata": {
        "id": "QGa8kMJ9pyha"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#output=pd.DataFrame()\n",
        "output=output.append(data,ignore_index=True)\n",
        "#output.columns=['positive_score','negative_score','Polarity_Score','subjectiivity_score','avg_senetence_length','Percentage_of_Complex_words','Fog_Index','avg_no_of_words_per_sentence','complex_Word_Count','word_count','syllable_count','personal_pronouns','avg_word_length']\n",
        "output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 488
        },
        "id": "DXp7kF2FExVD",
        "outputId": "f896883c-c391-4bfe-f057-176237cbe9f9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     positive_score  negative_score  Polarity_Score  subjectiivity_score  \\\n",
              "0              50.0            16.0        0.515152             0.023320   \n",
              "1              52.0            20.0        0.444444             0.018757   \n",
              "2              33.0            15.0        0.375000             0.011173   \n",
              "3              11.0             9.0        0.100000             0.002101   \n",
              "4              54.0            12.0        0.636364             0.022901   \n",
              "..              ...             ...             ...                  ...   \n",
              "145            74.0            38.0        0.321429             0.019481   \n",
              "146            40.0            18.0        0.379310             0.017742   \n",
              "147            38.0             4.0        0.809524             0.041262   \n",
              "148            56.0            10.0        0.696970             0.034150   \n",
              "149            47.0            41.0        0.068182             0.005687   \n",
              "\n",
              "     avg_senetence_length  Percentage_of_Complex_words   Fog_Index  \\\n",
              "0              156.214286                     0.274348   62.595454   \n",
              "1              134.960000                     0.269637   54.091855   \n",
              "2              146.533333                     0.214773   58.699243   \n",
              "3              139.829268                     0.298319   56.051035   \n",
              "4              120.604396                     0.284624   48.355608   \n",
              "..                    ...                          ...         ...   \n",
              "145            133.277778                     0.356602   53.453752   \n",
              "146            335.318182                     0.266935  134.234047   \n",
              "147            116.595238                     0.274272   46.747804   \n",
              "148            142.000000                     0.328879   56.931552   \n",
              "149             95.545455                     0.264455   38.323964   \n",
              "\n",
              "     avg_no_of_words_per_sentence  complex_Word_Count  word_count  \\\n",
              "0                       25.392857               400.0      1458.0   \n",
              "1                       22.266667               460.0      1706.0   \n",
              "2                       25.666667               346.0      1611.0   \n",
              "3                       22.512195               284.0       952.0   \n",
              "4                       19.615385               522.0      1834.0   \n",
              "..                            ...                 ...         ...   \n",
              "145                     20.033333               659.0      1848.0   \n",
              "146                     54.636364               331.0      1240.0   \n",
              "147                     19.523810               226.0       824.0   \n",
              "148                     22.066667               443.0      1347.0   \n",
              "149                     15.969697               279.0      1055.0   \n",
              "\n",
              "     syllable_count  personal_pronouns  avg_word_length  \n",
              "0            2785.0                6.0         5.063100  \n",
              "1            3313.0               21.0         4.998242  \n",
              "2            2864.0               35.0         4.538796  \n",
              "3            1853.0                2.0         5.095588  \n",
              "4            3533.0                3.0         5.060523  \n",
              "..              ...                ...              ...  \n",
              "145          4115.0                2.0         5.563853  \n",
              "146          2378.0                0.0         4.997581  \n",
              "147          1583.0               10.0         4.998786  \n",
              "148          2784.0                2.0         5.386785  \n",
              "149          2065.0                8.0         5.040758  \n",
              "\n",
              "[150 rows x 13 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3b7208f4-918c-4f34-9a4e-f421940d0b00\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>positive_score</th>\n",
              "      <th>negative_score</th>\n",
              "      <th>Polarity_Score</th>\n",
              "      <th>subjectiivity_score</th>\n",
              "      <th>avg_senetence_length</th>\n",
              "      <th>Percentage_of_Complex_words</th>\n",
              "      <th>Fog_Index</th>\n",
              "      <th>avg_no_of_words_per_sentence</th>\n",
              "      <th>complex_Word_Count</th>\n",
              "      <th>word_count</th>\n",
              "      <th>syllable_count</th>\n",
              "      <th>personal_pronouns</th>\n",
              "      <th>avg_word_length</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>50.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>0.515152</td>\n",
              "      <td>0.023320</td>\n",
              "      <td>156.214286</td>\n",
              "      <td>0.274348</td>\n",
              "      <td>62.595454</td>\n",
              "      <td>25.392857</td>\n",
              "      <td>400.0</td>\n",
              "      <td>1458.0</td>\n",
              "      <td>2785.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>5.063100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>52.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>0.444444</td>\n",
              "      <td>0.018757</td>\n",
              "      <td>134.960000</td>\n",
              "      <td>0.269637</td>\n",
              "      <td>54.091855</td>\n",
              "      <td>22.266667</td>\n",
              "      <td>460.0</td>\n",
              "      <td>1706.0</td>\n",
              "      <td>3313.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>4.998242</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>33.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>0.375000</td>\n",
              "      <td>0.011173</td>\n",
              "      <td>146.533333</td>\n",
              "      <td>0.214773</td>\n",
              "      <td>58.699243</td>\n",
              "      <td>25.666667</td>\n",
              "      <td>346.0</td>\n",
              "      <td>1611.0</td>\n",
              "      <td>2864.0</td>\n",
              "      <td>35.0</td>\n",
              "      <td>4.538796</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>11.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>0.100000</td>\n",
              "      <td>0.002101</td>\n",
              "      <td>139.829268</td>\n",
              "      <td>0.298319</td>\n",
              "      <td>56.051035</td>\n",
              "      <td>22.512195</td>\n",
              "      <td>284.0</td>\n",
              "      <td>952.0</td>\n",
              "      <td>1853.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.095588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>54.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>0.636364</td>\n",
              "      <td>0.022901</td>\n",
              "      <td>120.604396</td>\n",
              "      <td>0.284624</td>\n",
              "      <td>48.355608</td>\n",
              "      <td>19.615385</td>\n",
              "      <td>522.0</td>\n",
              "      <td>1834.0</td>\n",
              "      <td>3533.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>5.060523</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>145</th>\n",
              "      <td>74.0</td>\n",
              "      <td>38.0</td>\n",
              "      <td>0.321429</td>\n",
              "      <td>0.019481</td>\n",
              "      <td>133.277778</td>\n",
              "      <td>0.356602</td>\n",
              "      <td>53.453752</td>\n",
              "      <td>20.033333</td>\n",
              "      <td>659.0</td>\n",
              "      <td>1848.0</td>\n",
              "      <td>4115.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.563853</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>146</th>\n",
              "      <td>40.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>0.379310</td>\n",
              "      <td>0.017742</td>\n",
              "      <td>335.318182</td>\n",
              "      <td>0.266935</td>\n",
              "      <td>134.234047</td>\n",
              "      <td>54.636364</td>\n",
              "      <td>331.0</td>\n",
              "      <td>1240.0</td>\n",
              "      <td>2378.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.997581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>147</th>\n",
              "      <td>38.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>0.809524</td>\n",
              "      <td>0.041262</td>\n",
              "      <td>116.595238</td>\n",
              "      <td>0.274272</td>\n",
              "      <td>46.747804</td>\n",
              "      <td>19.523810</td>\n",
              "      <td>226.0</td>\n",
              "      <td>824.0</td>\n",
              "      <td>1583.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>4.998786</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>148</th>\n",
              "      <td>56.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>0.696970</td>\n",
              "      <td>0.034150</td>\n",
              "      <td>142.000000</td>\n",
              "      <td>0.328879</td>\n",
              "      <td>56.931552</td>\n",
              "      <td>22.066667</td>\n",
              "      <td>443.0</td>\n",
              "      <td>1347.0</td>\n",
              "      <td>2784.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.386785</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>149</th>\n",
              "      <td>47.0</td>\n",
              "      <td>41.0</td>\n",
              "      <td>0.068182</td>\n",
              "      <td>0.005687</td>\n",
              "      <td>95.545455</td>\n",
              "      <td>0.264455</td>\n",
              "      <td>38.323964</td>\n",
              "      <td>15.969697</td>\n",
              "      <td>279.0</td>\n",
              "      <td>1055.0</td>\n",
              "      <td>2065.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.040758</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>150 rows × 13 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3b7208f4-918c-4f34-9a4e-f421940d0b00')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3b7208f4-918c-4f34-9a4e-f421940d0b00 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3b7208f4-918c-4f34-9a4e-f421940d0b00');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8065
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#with open('output.csv', 'a') as f:#creating text file \n",
        " #   output.to_csv(f, index=False, header=False)\n",
        "#files.download('output.csv')"
      ],
      "metadata": {
        "id": "XcEXxciB6w0i"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}