text prepocecing to tugas akhir - NBC - UAS - 20220607.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "68cd80e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\ACER\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\ACER\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Mahasiswa memasukan library yang dibutuhkan dalam progaram text procecing ini.\n",
    "'''\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import LancasterStemmer\n",
    "from Sastrawi.Stemmer.StemmerFactory import StemmerFactory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a4b7ce9b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>No</th>\n",
       "      <th>from_user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>KMielva</td>\n",
       "      <td>@NSukahar @Dennysiregar7 @_ekokuntadhi @DivHum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>stellaayubi</td>\n",
       "      <td>RT @tiyaardiani: Gubernur Jabar Ridwan Kamil m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>MuriLesmana1</td>\n",
       "      <td>@detikcom Minyak goreng urusin yg bener...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>EmirSjarif</td>\n",
       "      <td>@dovityindah @dokterimun_id Yg dimakan apa? Co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>akufelfel</td>\n",
       "      <td>Yakni beras, jagung, bawang merah, cabai besar...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   No     from_user                                               text\n",
       "0   1       KMielva  @NSukahar @Dennysiregar7 @_ekokuntadhi @DivHum...\n",
       "1   2   stellaayubi  RT @tiyaardiani: Gubernur Jabar Ridwan Kamil m...\n",
       "2   3  MuriLesmana1         @detikcom Minyak goreng urusin yg bener...\n",
       "3   4    EmirSjarif  @dovityindah @dokterimun_id Yg dimakan apa? Co...\n",
       "4   5     akufelfel  Yakni beras, jagung, bawang merah, cabai besar..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Mahasiswa memasukan file excel kedalam program dengan menggunakan read_excel\n",
    "'''\n",
    "df = pd.read_excel('Tweet_Minyak_Goreng.xlsx', header=0)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8c356944",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lwr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@nsukahar @dennysiregar7 @_ekokuntadhi @divhum...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rt @tiyaardiani: gubernur jabar ridwan kamil m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>@detikcom minyak goreng urusin yg bener...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>@dovityindah @dokterimun_id yg dimakan apa? co...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>yakni beras, jagung, bawang merah, cabai besar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>558</th>\n",
       "      <td>blt minyak goreng untuk jaga daya beli masyara...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>pastikan ketersedian minyak goreng, kapolres j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>blt minyak goreng rp 300.000 bakal cair\\nayo p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>kementerian perindustrian, asosiasi pedagang p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>bumn holding pangan telah mendistribusikan min...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>563 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   lwr\n",
       "0    @nsukahar @dennysiregar7 @_ekokuntadhi @divhum...\n",
       "1    rt @tiyaardiani: gubernur jabar ridwan kamil m...\n",
       "2           @detikcom minyak goreng urusin yg bener...\n",
       "3    @dovityindah @dokterimun_id yg dimakan apa? co...\n",
       "4    yakni beras, jagung, bawang merah, cabai besar...\n",
       "..                                                 ...\n",
       "558  blt minyak goreng untuk jaga daya beli masyara...\n",
       "559  pastikan ketersedian minyak goreng, kapolres j...\n",
       "560  blt minyak goreng rp 300.000 bakal cair\\nayo p...\n",
       "561  kementerian perindustrian, asosiasi pedagang p...\n",
       "562  bumn holding pangan telah mendistribusikan min...\n",
       "\n",
       "[563 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Mahasiswa merubah semua kata menjadi huruf kecil atau lower.\n",
    "'''\n",
    "def clean_lower(lwr):\n",
    "    lwr = lwr.lower()\n",
    "    return lwr\n",
    "\n",
    "df['lwr'] = df['text'].apply(clean_lower)\n",
    "casefolding = pd.DataFrame(df['lwr'])\n",
    "casefolding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cbf3fb07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       nsukahar  dennysiregar7   ekokuntadhi  divhum...\n",
       "1      rt  tiyaardiani  gubernur jabar ridwan kamil m...\n",
       "2              detikcom minyak goreng urusin yg bener   \n",
       "3       dovityindah  dokterimun id yg dimakan apa  co...\n",
       "4      yakni beras jagung bawang merah cabai besar ca...\n",
       "                             ...                        \n",
       "558    blt minyak goreng untuk jaga daya beli masyara...\n",
       "559    pastikan ketersedian minyak goreng kapolres ja...\n",
       "560    blt minyak goreng rp 300 000 bakal cair ayo pe...\n",
       "561    kementerian perindustrian asosiasi pedagang pa...\n",
       "562    bumn holding pangan telah mendistribusikan min...\n",
       "Name: clean_punct, Length: 563, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Remove Puncutuation\n",
    "clean_spcl = re.compile('[/(){}\\[\\]\\|,;]')\n",
    "clean_symbol = re.compile('[^0-9a-z]')\n",
    "def clean_punct(text):\n",
    "    text = clean_spcl.sub('', text)\n",
    "    text = clean_symbol.sub(' ', text)\n",
    "    return text# Buat kolom tambahan untuk data description yang telah diremovepunctuation   \n",
    "df['clean_punct'] = df['lwr'].apply(clean_punct)\n",
    "df['clean_punct']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20a389d8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      nsukahar dennysiregar7 ekokuntadhi divhumas po...\n",
       "1      rt tiyaardiani gubernur jabar ridwan kamil mel...\n",
       "2                 detikcom minyak goreng urusin yg bener\n",
       "3      dovityindah dokterimun id yg dimakan apa coba ...\n",
       "4      yakni beras jagung bawang merah cabai besar ca...\n",
       "                             ...                        \n",
       "558    blt minyak goreng untuk jaga daya beli masyara...\n",
       "559    pastikan ketersedian minyak goreng kapolres ja...\n",
       "560    blt minyak goreng rp 300 000 bakal cair ayo pe...\n",
       "561    kementerian perindustrian asosiasi pedagang pa...\n",
       "562    bumn holding pangan telah mendistribusikan min...\n",
       "Name: clean_double_ws, Length: 563, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def _normalize_whitespace(text):\n",
    "    corrected = str(text)\n",
    "    corrected = re.sub(r\"//t\",r\"\\t\", corrected)\n",
    "    corrected = re.sub(r\"( )\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\n)\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\r)\\1+\",r\"\\1\", corrected)\n",
    "    corrected = re.sub(r\"(\\t)\\1+\",r\"\\1\", corrected)\n",
    "    return corrected.strip(\" \")\n",
    "df['clean_double_ws'] = df['clean_punct'].apply(_normalize_whitespace)\n",
    "df['clean_double_ws']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "749c0227",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      nsukahar dennysiregar7 ekokuntadhi divhumas po...\n",
       "1      tiyaardiani gubernur jabar ridwan kamil melepa...\n",
       "2                             minyak goreng urusin bener\n",
       "3      dovityindah dokterimun id dimakan coba makan t...\n",
       "4      beras jagung bawang merah cabai cabai rawit da...\n",
       "                             ...                        \n",
       "558    blt minyak goreng jaga daya beli masyarakat mi...\n",
       "559    pastikan ketersedian minyak goreng kapolres ja...\n",
       "560    blt minyak goreng rp 300 000 cair ayo percaya ...\n",
       "561    kementerian perindustrian asosiasi pedagang pa...\n",
       "562    bumn holding pangan mendistribusikan minyak go...\n",
       "Name: clean_sw, Length: 563, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_stopwords = stopwords.words('indonesian')\n",
    "\n",
    "\n",
    "# ---------------------------- manualy add stopword  ------------------------------------\n",
    "# append additional stopword\n",
    "list_stopwords.extend([\"yg\", \"dg\", \"rt\", \"dgn\", \"ny\", \"d\", 'klo', \n",
    "                       'kalo', 'amp', 'biar', 'bikin', 'bilang', \n",
    "                       'gak', 'ga', 'krn', 'nya', 'nih', 'sih', \n",
    "                       'si', 'tau', 'tdk', 'tuh', 'utk', 'ya', \n",
    "                       'jd', 'jgn', 'sdh', 'aja', 'n', 't', \n",
    "                       'nyg', 'hehe', 'pen', 'u', 'nan', 'loh', 'rt',\n",
    "                       '&amp', 'yah', 'https', 'detikcom'])\n",
    "\n",
    "\n",
    "# convert list to dictionary\n",
    "list_stopwords = set(list_stopwords)\n",
    "\n",
    "def clean_stopwords(text):\n",
    "    text = ' '.join(word for word in text.split() if word not in list_stopwords) # hapus stopword dari kolom deskripsi\n",
    "    return text# Buat kolom tambahan untuk data description yang telah distopwordsremoval   \n",
    "\n",
    "df['clean_sw'] = df['clean_double_ws'].apply(clean_stopwords)\n",
    "df['clean_sw']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d0cd2850",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      nsukahar dennysiregar7 ekokuntadhi divhumas po...\n",
       "1      tiyaardiani gubernur jabar ridwan kamil lepas ...\n",
       "2                             minyak goreng urusin bener\n",
       "3      dovityindah dokterimun id makan coba makan tep...\n",
       "4      beras jagung bawang merah cabai cabai rawit da...\n",
       "                             ...                        \n",
       "558    blt minyak goreng jaga daya beli masyarakat mi...\n",
       "559    pasti dian minyak goreng kapolres jaktim kunju...\n",
       "560    blt minyak goreng rp 300 000 cair ayo percaya ...\n",
       "561    menteri industri asosiasi dagang pasar mang pe...\n",
       "562    bumn holding pangan distribusi minyak goreng p...\n",
       "Name: sudah_bersih, Length: 563, dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "factory = StemmerFactory()\n",
    "stemmer = factory.create_stemmer()\n",
    "\n",
    "df['sudah_bersih'] = df['clean_sw'].apply(stemmer.stem)\n",
    "df['sudah_bersih']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "266e4286",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(\"token nbc tugas akhir.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e23e0cd0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "c42d99d755d00efaf066b964ceed011a4027ae5af520c8ef8190b9acee3a6094"
  },
  "kernelspec": {
   "display_name": "Python 3.9.4 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}