full_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\batyo\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\batyo\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: emoji==1.4.2 in d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages (1.4.2)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n",
      "WARNING: Ignoring invalid distribution -orch (d:\\programing\\programs\\anaconda\\envs\\pytorch\\lib\\site-packages)\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "import string\n",
    "import re\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "!pip install emoji==1.4.2\n",
    "import emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean/Normalize Arabic Text \n",
    "# taskeel, longation, ... etc\n",
    "def clean_str(text):\n",
    "    search = [\"أ\",\"إ\",\"آ\",\"ة\",\"_\",\"-\",\"/\",\".\",\"،\",\" و \",\" يا \",'\"',\"ـ\",\"'\",\"ى\",\"\\\\\",'\\n', '\\t','\"','?','؟','!']\n",
    "    replace = [\"ا\",\"ا\",\"ا\",\"ه\",\" \",\" \",\"\",\"\",\"\",\" و\",\" يا\",\"\",\"\",\"\",\"ي\",\"\",' ', ' ',' ',' ? ',' ؟ ',' ! ']\n",
    "    \n",
    "    #remove tashkeel\n",
    "    p_tashkeel = re.compile(r'[\\u0617-\\u061A\\u064B-\\u0652]')\n",
    "    text = re.sub(p_tashkeel,\"\", text)\n",
    "    \n",
    "    #remove longation\n",
    "    p_longation = re.compile(r'(.)\\1+')\n",
    "    subst = r\"\\1\\1\"\n",
    "    text = re.sub(p_longation, subst, text)\n",
    "    \n",
    "    text = text.replace('وو', 'و')\n",
    "    text = text.replace('يي', 'ي')\n",
    "    text = text.replace('اا', 'ا')\n",
    "    \n",
    "    for i in range(0, len(search)):\n",
    "        text = text.replace(search[i], replace[i])\n",
    "    \n",
    "    #trim    \n",
    "    text = text.strip()\n",
    "\n",
    "    return text\n",
    "\n",
    "def PreProcessing(text, unk='<UNK>'):\n",
    "\n",
    "  for i in range(len(text)):\n",
    "\n",
    "    # remove links\n",
    "    text[i] = re.sub(r'https?:\\/\\/.*[\\r\\n]*', '', text[i], flags=re.MULTILINE)\n",
    "\n",
    "    # limit repeated emojis to 2\n",
    "    prev_emoji = None\n",
    "    cnt = 0\n",
    "    remove_duplicate_emoji = []\n",
    "    for c in text[i]:\n",
    "        if c in emoji.UNICODE_EMOJI['en']:\n",
    "            if prev_emoji == c and cnt >=2:\n",
    "                continue\n",
    "            if prev_emoji != c:\n",
    "              cnt = 0\n",
    "            prev_emoji = c\n",
    "            cnt+=1\n",
    "        remove_duplicate_emoji.append(c)\n",
    "    text[i] = \"\".join(remove_duplicate_emoji)\n",
    "\n",
    "    # seperate emojis\n",
    "    text[i] = ''.join((' '+c+' ') if c in emoji.UNICODE_EMOJI['en'] else c for c in text[i])\n",
    "\n",
    "    # Clean/Normalize Arabic Text\n",
    "    text[i] = clean_str(text[i]) \n",
    "\n",
    "    # remove english words\n",
    "    text[i] = re.sub(r'\\s*[A-Za-z]+\\b', '' , text[i])\n",
    "\n",
    "    # remove numbers\n",
    "    text[i] = re.sub(\"\\d+\", \" \", text[i])\n",
    "\n",
    "    # remove punctuations\n",
    "    text[i] = text[i].translate(str.maketrans('', '', string.punctuation))\n",
    "\n",
    "    # tokenize\n",
    "    text[i] = nltk.tokenize.word_tokenize(text[i])\n",
    "\n",
    "    # remove stop-words\n",
    "    stopwords = set(nltk.corpus.stopwords.words(\"arabic\"))\n",
    "    text[i] = [word for word in text[i] if word not in stopwords]\n",
    "\n",
    "    # remove emojis\n",
    "    text[i] = [word for word in text[i] if word not in emoji.UNICODE_EMOJI['en']]\n",
    "\n",
    "    # remove empty words\n",
    "    text[i] = [word for word in text[i] if len(word)>2]\n",
    "\n",
    "    # if the tweet became empty just add add <UNK> token \n",
    "    if len(text[i])==0:\n",
    "      text[i] = [unk]\n",
    "\n",
    "  return text"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "f707ed687e1cc7dca614d866740125e744cc3f7963ec2d63a60d682146be2e45"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}