ArabicTextDiacritization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From c:\\Users\\hlaha\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import pyarabic.araby as araby\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils import to_categorical\n",
    "import farasa\n",
    "from farasa.segmenter import FarasaSegmenter \n",
    "import unicodedata\n",
    "import torch\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Embedding, Bidirectional, LSTM, Dense, Dropout, TimeDistributed, BatchNormalization\n",
    "from tensorflow.keras.optimizers import Adamax\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from tensorflow.keras.activations import relu,linear\n",
    "\n",
    "from gensim.models.word2vec import LineSentence\n",
    "from gensim.models import FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Run on GPU\n",
    "# use_cuda = torch.cuda.is_available()\n",
    "# device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "# print (device)\n",
    "# # print the cpu or gpu\n",
    "# print(torch.cuda.get_device_name(0))\n",
    "# # print the number of gpus you have\n",
    "# print(torch.cuda.device_count())\n",
    "# # print current gpu\n",
    "# print(torch.cuda.current_device())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(file_path):\n",
    "    \"\"\"\n",
    "    Read the contents of the file located at file_path \n",
    "    and append each line to the list data\n",
    "    \"\"\"\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        data = file.readlines()\n",
    "\n",
    "        # remove '\\n' from each line\n",
    "        data = [line.strip() for line in data]\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_before_preprocessing = read_data(\"./dataset/train.txt\")\n",
    "print(len(data_before_preprocessing))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_pickle_file(file_path):\n",
    "    \"\"\"\n",
    "    Read the contents of the pickle file located at file_path \n",
    "    and append each line to the list data\n",
    "    \"\"\"\n",
    "    with open(file_path, 'rb') as file:\n",
    "        data = pickle.load(file)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_words_in_file(path, words, permission='w'):\n",
    "    \"\"\"\n",
    "    Save the words in the file located at path \n",
    "    \"\"\"\n",
    "    with open(path, permission, encoding='utf-8') as file:\n",
    "            for word in words:\n",
    "                file.write(word + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36\n",
      "{'ت', 'ن', 'س', 'ى', 'غ', 'ق', 'ح', 'ث', 'ظ', 'د', 'ش', 'ة', 'ؤ', 'ص', 'ب', 'ط', 'ذ', 'أ', 'إ', 'ك', 'ا', 'ز', 'خ', 'ع', 'ر', 'ي', 'ئ', 'ه', 'م', 'ل', 'ء', 'و', 'ض', 'ج', 'ف', 'آ'}\n"
     ]
    }
   ],
   "source": [
    "# set for arabic letters\n",
    "arabic_letters = set(read_pickle_file(\"./Delivery/arabic_letters.pickle\"))\n",
    "\n",
    "print(len(arabic_letters))\n",
    "print(arabic_letters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "{'ِ', 'ً', 'ٌ', 'ُ', 'َ', 'ْ', 'ّ', 'ٍ'}\n"
     ]
    }
   ],
   "source": [
    "# set for arabic letters\n",
    "diacritics = set(read_pickle_file(\"./Delivery/diacritics.pickle\"))\n",
    "\n",
    "print(len(diacritics))\n",
    "print(diacritics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove diacritics\n",
    "def remove_diacritics(text):\n",
    "    text = araby.strip_tashkeel(text)\n",
    "    return text\n",
    "\n",
    "# Remove any letters not found in set arabic_letters and not found in set diacritics\n",
    "def remove_non_arabic(text):\n",
    "    text = re.sub(r'[^\\s' + ''.join(arabic_letters) + ''.join(diacritics) + ']', '', text)\n",
    "    return text\n",
    "\n",
    "def input_preprocessing_text(text):\n",
    "    # Correct most common errors on word like repetetion of harakats, or tanween before alef\n",
    "    text = araby.autocorrect(text)\n",
    "\n",
    "    # Remove any non-Arabic letters\n",
    "    text = remove_non_arabic(text)\n",
    "\n",
    "    # Remove diacritics\n",
    "    text = remove_diacritics(text)\n",
    "\n",
    "    # Tokenize\n",
    "    text = araby.tokenize(text)\n",
    "\n",
    "    return text\n",
    "\n",
    "def save_tokenized_input(text,path=\"./generatedFiles/training/tokenized_input.txt\", permission='w'):\n",
    "    words = input_preprocessing_text(text)\n",
    "    save_words_in_file(path, words, permission)\n",
    "    \n",
    "\n",
    "def save_gold_output(text,path=\"./generatedFiles/training/gold_output.txt\", permission='w'):\n",
    "    # Remove any non-Arabic letters and extra spaces\n",
    "    text = remove_non_arabic(text)\n",
    "\n",
    "    # Tokenize\n",
    "    text = araby.tokenize(text)\n",
    "\n",
    "    save_words_in_file(path, text, permission)\n",
    "\n",
    "\n",
    "def is_not_arabic_diacritic(char):\n",
    "   category = unicodedata.category(char)\n",
    "   return not (category == 'Mn' or category == 'Mc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_sentence_in_file(path, words, permission='w'):\n",
    "    \"\"\"\n",
    "    Save the words in the file located at path \n",
    "    \"\"\"\n",
    "    with open(path, permission, encoding='utf-8') as file:\n",
    "            file.write(words + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_new_input_sentence(text,path=\"./generatedFiles/training/new_input_sentence.txt\", permission='w'):\n",
    "    # Remove any non-Arabic letters and extra spaces\n",
    "    text = remove_non_arabic(text)\n",
    "    # Remove diacritics\n",
    "    text = remove_diacritics(text)\n",
    "    \n",
    "    #remove extra spaces between words\n",
    "    text = re.sub(r'\\s+', ' ', text)\n",
    "\n",
    "    save_sentence_in_file(path, text, permission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Example usage:\n",
    "# character = 'ذْ'\n",
    "# if is_not_arabic_diacritic(character[1]):\n",
    "#    print(\"The character is not an Arabic diacritic.\")\n",
    "# else:\n",
    "#    print(\"The character is an Arabic diacritic.\")\n",
    "\n",
    "\n",
    "# # Testing of is_not_arabic_diacritic() function with gettting the index of the first non diacritic character in the word\n",
    "# word = 'زَّراع'\n",
    " \n",
    "# for i in range(1, len(word)): # start from 1 because the first character is not a diacritic\n",
    "#     if is_not_arabic_diacritic(word[i]):\n",
    "#         print(i)\n",
    "#         break "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # RUN ONE TIME ONLY THIS CODE AGAIN \n",
    "# # Generate Gold Input file\n",
    "# for i in range(len(data_before_preprocessing)):\n",
    "#     save_tokenized_input(data_before_preprocessing[i], permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # RUN ONE TIME ONLY THIS CODE AGAIN\n",
    "# # Generate Gold Output file\n",
    "# for i in range(len(data_before_preprocessing)):\n",
    "#     save_gold_output(data_before_preprocessing[i], permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # RUN ONE TIME ONLY THIS CODE AGAIN\n",
    "# # Generate input Sentence file\n",
    "# for i in range(len(data_before_preprocessing)):\n",
    "#     save_new_input_sentence(data_before_preprocessing[i], permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Important functions in PyArabic\n",
    "\n",
    "# araby.tokenize(text) # Tokenize the sentence text into words\n",
    "# araby.is_arabicrange(text) # Check if the text is Arabic\n",
    "# araby.sentence_tokenize(text) # Tokenize the text into sentences\n",
    "# araby.strip_tashkeel(text) # Remove diacritics (FATHA, DAMMA, KASRA, SUKUN, SHADDA, FATHATAN, DAMMATAN, KASRATAN)\n",
    "# araby.strip_diacritics(text) # Remove diacritics (Small Alef الألف الخنجرية, Harakat + Shadda, Quranic marks)\n",
    "# araby.strip_tatweel(text) # Remove tatweel\n",
    "# araby.strip_shadda(text) # Remove shadda\n",
    "# araby.autocorrect(text) # Correct most common errors on word like repetetion of harakats,or tanwin befor alef\n",
    "# araby.arabicrange() # Return a list of Arabic characters\n",
    "\n",
    "# New Functions in PyArabic\n",
    "# araby.vocalized_similarity(word1, word2) # if the two words has the same letters and the same harakats, this function return True. \n",
    "# The two words can be full vocalized, or partial vocalized\n",
    "\n",
    "# araby.vocalizedlike(word1, word2) Same as vocalized_similarity but return True and False\n",
    "\n",
    "# araby.joint(word1, word2) # joint the letters with the marks the length ot letters and marks must be equal return word\n",
    "\n",
    "\n",
    "\n",
    "# Return the text, its tashkeel and shadda if extract_shadda is True\n",
    "# text, marks, shada = araby.separate(text,extract_shadda=True) # Separate diacritics from the text\n",
    "# print (text)\n",
    "# for m in marks:\n",
    "#     print (araby.name(m))\n",
    "\n",
    "# for s in shada:\n",
    "#     print (araby.name(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['قوله', 'أو', 'قطع', 'الأول', 'يده', 'إلخ', 'قال', 'الزركشي', 'ابن', 'عرفة']\n",
      "2101983\n"
     ]
    }
   ],
   "source": [
    "# Read tokenized_input file\n",
    "tokenized_input = read_data(\"./generatedFiles/training/tokenized_input.txt\")\n",
    "print(tokenized_input[:10])\n",
    "print(len(tokenized_input))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Core Word (CW) Diacritization**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Feature Extraction**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 - Characters: \n",
    "Here we extract each character from all tokenized words and create a vector of size 50 for each character."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_char = Tokenizer(char_level=True)\n",
    "tokenizer_char.fit_on_texts(tokenized_input)\n",
    "sequences_char = tokenizer_char.texts_to_sequences(tokenized_input)\n",
    "char_features = pad_sequences(sequences_char)   # padding the sequences to have the same length as the longest sequence (word)\n",
    "char_embeddings = np.random.rand(len(tokenizer_char.word_index) + 1, embedding_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the tokenizer_char model\n",
    "with open('./generatedFiles/training/tokenizer_char.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer_char, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the sequences_char model\n",
    "with open('./generatedFiles/training/sequences_char.pickle', 'wb') as handle:\n",
    "    pickle.dump(sequences_char, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the char_embeddings array in a pickle file\n",
    "with open('./generatedFiles/training/char_embeddings.pickle', 'wb') as handle:\n",
    "    pickle.dump(char_embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the char_features array in a pickle file\n",
    "with open('./generatedFiles/training/char_features.pickle', 'wb') as handle:\n",
    "    pickle.dump(char_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2101983, 13)\n",
      "(37, 40)\n"
     ]
    }
   ],
   "source": [
    "print(char_features.shape) # (number of words, max length of word in the dataset)\n",
    "\n",
    "\n",
    "print(char_embeddings.shape)\n",
    "\n",
    "# 38 rows: 37 unique characters identified by the tokenizer, 1 row for handling characters not seen in the training data\n",
    "# 50 columns: Each character is encoded as a 50-dimensional vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  0  0  0  0  0  0  0  0 13  5  1  7]\n"
     ]
    }
   ],
   "source": [
    "print(char_features[0]) \n",
    "# the number of non zero elements corresponds to the length of the word \n",
    "# and the value of each element corresponds to the index of the character in the tokenizer\n",
    "# which means that every character now is encoded as a number and this number is the index of the character in the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      " 0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      " 0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      " 0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      " 0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      " 0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      " 0.08634157 0.4526868  0.5898067  0.44434528]\n"
     ]
    }
   ],
   "source": [
    "print(char_embeddings[0])\n",
    "# this is the embedding of each character in the tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 40)\n",
      "[[0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.80871371 0.12394689 0.87569474 0.31353818 0.33009045 0.11812771\n",
      "  0.38956887 0.62473864 0.43774451 0.05983604 0.79880199 0.98974258\n",
      "  0.23386662 0.25450462 0.49377536 0.6882848  0.73133686 0.62197055\n",
      "  0.92977178 0.82167678 0.91133812 0.64268308 0.64488677 0.14072622\n",
      "  0.54441718 0.31470806 0.62167814 0.8099589  0.47332709 0.37057716\n",
      "  0.6861456  0.411859   0.58265002 0.92869714 0.36860802 0.49624959\n",
      "  0.08634157 0.4526868  0.5898067  0.44434528]\n",
      " [0.79791793 0.80401018 0.7096797  0.35601843 0.02583895 0.81833114\n",
      "  0.83543821 0.10887958 0.70178611 0.18971199 0.93066788 0.25161477\n",
      "  0.63959308 0.77035551 0.67635974 0.58321519 0.12935825 0.55489913\n",
      "  0.30778296 0.80051817 0.28731892 0.35279727 0.01928517 0.19074971\n",
      "  0.77645134 0.22707999 0.02490547 0.44530131 0.51715747 0.89385559\n",
      "  0.7905519  0.95851561 0.52063615 0.22194732 0.21318093 0.91040893\n",
      "  0.31841845 0.82440046 0.86374199 0.13732761]\n",
      " [0.58786353 0.36178631 0.23029876 0.75367712 0.78879611 0.54356207\n",
      "  0.30758601 0.83894193 0.12935863 0.42913136 0.57190114 0.39638332\n",
      "  0.56109088 0.93151887 0.95497757 0.19307364 0.03617345 0.17160821\n",
      "  0.58669999 0.14381791 0.98064519 0.40775706 0.26062596 0.14192332\n",
      "  0.53891232 0.99480954 0.84515416 0.62273199 0.06131796 0.9881755\n",
      "  0.4708029  0.59122424 0.53232905 0.00407433 0.21474428 0.78097428\n",
      "  0.06029918 0.75889144 0.11160706 0.51833428]\n",
      " [0.89564725 0.25195525 0.07590479 0.24319147 0.71416636 0.35021229\n",
      "  0.91162322 0.55128277 0.72244724 0.67865803 0.5007849  0.74818395\n",
      "  0.39703012 0.60525603 0.30358701 0.65201726 0.27057574 0.68505244\n",
      "  0.12685006 0.44511192 0.98633195 0.93323719 0.31900607 0.77598429\n",
      "  0.93693782 0.78101963 0.1549614  0.37614095 0.99988996 0.88039479\n",
      "  0.29993726 0.45198818 0.59049471 0.35870078 0.49778056 0.05134592\n",
      "  0.63903589 0.58398972 0.8917426  0.44850799]\n",
      " [0.30590491 0.28927431 0.31589861 0.42320636 0.07978032 0.00334554\n",
      "  0.14534933 0.19105615 0.6192867  0.29012318 0.33833394 0.89595105\n",
      "  0.88263227 0.40273384 0.43214777 0.7322524  0.88394433 0.92725942\n",
      "  0.5108283  0.36694897 0.38792433 0.7839541  0.49452916 0.94370478\n",
      "  0.65390249 0.17345121 0.92727838 0.35063832 0.28414709 0.51371473\n",
      "  0.15315263 0.18277153 0.46026628 0.747792   0.1078833  0.80023742\n",
      "  0.87861832 0.78322987 0.91420281 0.21658773]]\n"
     ]
    }
   ],
   "source": [
    "print(char_embeddings[char_features[0]].shape)\n",
    "# 13 is the word of characters and 50 is the embedding size of each character\n",
    "\n",
    "print(char_embeddings[char_features[0]])\n",
    "# this is the embedding of each character in the first tokenized word, this is the 1st feature and the input of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.89564725 0.25195525 0.07590479 0.24319147 0.71416636 0.35021229\n",
      " 0.91162322 0.55128277 0.72244724 0.67865803 0.5007849  0.74818395\n",
      " 0.39703012 0.60525603 0.30358701 0.65201726 0.27057574 0.68505244\n",
      " 0.12685006 0.44511192 0.98633195 0.93323719 0.31900607 0.77598429\n",
      " 0.93693782 0.78101963 0.1549614  0.37614095 0.99988996 0.88039479\n",
      " 0.29993726 0.45198818 0.59049471 0.35870078 0.49778056 0.05134592\n",
      " 0.63903589 0.58398972 0.8917426  0.44850799]\n",
      "[0.58786353 0.36178631 0.23029876 0.75367712 0.78879611 0.54356207\n",
      " 0.30758601 0.83894193 0.12935863 0.42913136 0.57190114 0.39638332\n",
      " 0.56109088 0.93151887 0.95497757 0.19307364 0.03617345 0.17160821\n",
      " 0.58669999 0.14381791 0.98064519 0.40775706 0.26062596 0.14192332\n",
      " 0.53891232 0.99480954 0.84515416 0.62273199 0.06131796 0.9881755\n",
      " 0.4708029  0.59122424 0.53232905 0.00407433 0.21474428 0.78097428\n",
      " 0.06029918 0.75889144 0.11160706 0.51833428]\n"
     ]
    }
   ],
   "source": [
    "print(char_embeddings[1])\n",
    "print(char_embeddings[5])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 - The position of the character in a word segment:\n",
    "For example, given the word “wAlktAb” , which is composed of three segments “w+Al+ktAb”. Letters were marked as “B” if they begin a segment, “M” if they are in the middle of a segment, “E” if they end a segment, and “S” if they are single letter segments. So for “w+Al+ktAb”, the corresponding character positions are “S+BE+BMME.”"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2024-01-02 16:30:29,710 - farasapy_logger - WARNING]: Be careful with large lines as they may break on interactive mode. You may switch to Standalone mode for such cases.\n"
     ]
    }
   ],
   "source": [
    "segmenter = FarasaSegmenter(interactive=True) # The default behaviour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_seg_tags(word):                 # word = \"wAlktAb\"\n",
    "    segments = segmenter.segment(word)  # segments will be a list: [\"w\", \"Al\", \"ktAb\"]\n",
    "    segments = segments.split('+')\n",
    "    seg_tags = []\n",
    "    for segment in segments:\n",
    "        if len(segment) == 1:\n",
    "            seg_tags.append(\"S\")\n",
    "        else:\n",
    "            seg_tags.append(\"B\")  # First letter\n",
    "            seg_tags.extend(\"M\" * (len(segment) - 2))  # Middle letters\n",
    "            seg_tags.append(\"E\")  # Last letter\n",
    "    return segments, seg_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word = \"كقلمه\"\n",
    "# segments, seg_tags = get_seg_tags(word)\n",
    "# print(\"Segmented word:\", segments)\n",
    "# print(\"SEG tags:\", seg_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # DON'T RUN THIS CODE AGAIN, THIS CELL TOOK 25m 4.5s TO RUN\n",
    "# # The Output of this code is the input_segments.txt file\n",
    "\n",
    "# for i in range(len(tokenized_input)):\n",
    "#     segments, seg_tags = get_seg_tags(tokenized_input[i])\n",
    "#     # Write and append on the tokenized input to a file\n",
    "#     with open('./generatedFiles/training/input_segments.txt', 'a', encoding='utf-8') as file:\n",
    "#         for tag in seg_tags:\n",
    "#             file.write(tag)\n",
    "#         file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2101983\n",
      "['BMES', 'BE', 'BME', 'BEBME', 'BES', 'BME', 'BME', 'BEBMMME', 'BME', 'BMES']\n"
     ]
    }
   ],
   "source": [
    "input_segments = read_data(\"./generatedFiles/training/input_segments.txt\")\n",
    "print(len(input_segments))\n",
    "print(input_segments[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer_tags = Tokenizer(char_level=True)\n",
    "tokenizer_tags.fit_on_texts(input_segments)\n",
    "sequences_tags = tokenizer_tags.texts_to_sequences(input_segments)\n",
    "tags_features = pad_sequences(sequences_tags)   \n",
    "tags_embeddings = np.random.rand(len(tokenizer_tags.word_index) + 1, embedding_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the tokenizer_tags model\n",
    "with open('./generatedFiles/training/tokenizer_tags.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer_tags, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the sequences_char model\n",
    "with open('./generatedFiles/training/sequences_tags.pickle', 'wb') as handle:\n",
    "    pickle.dump(sequences_tags, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the tags_embeddings array in a pickle file\n",
    "with open('./generatedFiles/training/tags_embeddings.pickle', 'wb') as handle:\n",
    "    pickle.dump(tags_embeddings, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# Save the tags_features array in a pickle file\n",
    "with open('./generatedFiles/training/tags_features.pickle', 'wb') as handle:\n",
    "    pickle.dump(tags_features, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tags_features.shape) \n",
    "print(tags_embeddings.shape)\n",
    "tags_features[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_embeddings[tags_features[0][0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3 - PRIOR: \n",
    "diacritics seen in the training set per segment. Since we used a character-level model, this feature informed the model with word-level information. For example, the word “ktAb”  was observed to have two diacritized forms in the training set, namely “kitaAb” ( – book) and “kut∼aAb” ( – writers). The first letter in the word (“k”) accepted the diacritics “i” and “u.” Thus, given a binary vector representing whether a character is allowed to assume any of the eight primitive Arabic diacritic marks (a, i, u, o, K, N, F, and ∼ in order), the first letter would be given the following vector “01100000.” If a word segment was never observed during training, then the vector for all letters therein would be set to 11111111."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_output = read_data(\"./generatedFiles/training/gold_output.txt\")\n",
    "print(len(gold_output))\n",
    "print(gold_output[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Map each diacrtics to its unicode\n",
    "diacritics_mapping = {\n",
    "    'FATHA': '\\u064E',\n",
    "    'DAMMA': '\\u064F',\n",
    "    'KASRA': '\\u0650',\n",
    "    'SHADDA': '\\u0651',\n",
    "    'SUKUN': '\\u0652',\n",
    "    'FATHATAN': '\\u064B',\n",
    "    'DAMMATAN': '\\u064C',\n",
    "    'KASRATAN': '\\u064D'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Extract diacritics by returning a list containing a tuple of 3 elements: (letter, tashkeel, shadda)\n",
    "# def extract_arabic_diacritics(word):\n",
    "#     diacritics_list = []\n",
    "#     extracted_word, tashkeel, shadda = araby.separate(word, extract_shadda=True)\n",
    "#     for i in range(len(extracted_word)):\n",
    "#         print(f'{araby.name(extracted_word[i])} {araby.name(tashkeel[i])} {araby.name(shadda[i])}')\n",
    "#         diacritics_list.append((extracted_word[i], (tashkeel[i].encode(\"utf8\")).decode(), (shadda[i].encode(\"utf8\")).decode()))\n",
    "#     return diacritics_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# letter, tashkeel, shadda = araby.separate('زَّ', extract_shadda=True)   # SHADDA + FATHA Example\n",
    "\n",
    "# enkar = 'كَإِنْكَارِ'\n",
    "\n",
    "# print('FATHA in tashkeel: ', diacritics_mapping['FATHA'] in tashkeel)\n",
    "# print('DAMMA in tashkeel: ', diacritics_mapping['DAMMA'] in tashkeel)\n",
    "# print('KASRA in tashkeel: ', diacritics_mapping['KASRA'] in tashkeel)\n",
    "# print('SUKUN in tashkeel: ', diacritics_mapping['SUKUN'] in tashkeel)\n",
    "# print('FATHATAN in tashkeel: ', diacritics_mapping['FATHATAN'] in tashkeel)\n",
    "# print('DAMMATAN in tashkeel: ', diacritics_mapping['DAMMATAN'] in tashkeel)\n",
    "# print('KASRATAN in tashkeel: ', diacritics_mapping['KASRATAN'] in tashkeel)\n",
    "# print('SHADDA in tashkeel: ', diacritics_mapping['SHADDA'] in tashkeel)\n",
    "# print('=============================')\n",
    "# print('FATHA in shadda: ', diacritics_mapping['FATHA'] in shadda)\n",
    "# print('DAMMA in shadda: ', diacritics_mapping['DAMMA'] in shadda)\n",
    "# print('KASRA in shadda: ', diacritics_mapping['KASRA'] in shadda)\n",
    "# print('SUKUN in shadda: ', diacritics_mapping['SUKUN'] in shadda)\n",
    "# print('FATHATAN in shadda: ', diacritics_mapping['FATHATAN'] in shadda)\n",
    "# print('DAMMATAN in shadda: ', diacritics_mapping['DAMMATAN'] in shadda)\n",
    "# print('KASRATAN in shadda: ', diacritics_mapping['KASRATAN'] in shadda)\n",
    "# print('SHADDA in shadda: ', diacritics_mapping['SHADDA'] in shadda)\n",
    "\n",
    "# print((diacritics_mapping['SUKUN'] in tashkeel and diacritics_mapping['FATHA'] not in tashkeel and diacritics_mapping['DAMMA'] not in tashkeel and diacritics_mapping['KASRA'] not in tashkeel))\n",
    "# print((diacritics_mapping['SUKUN'] in tashkeel and diacritics_mapping['SHADDA'] not in shadda))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# firstly, initialize an empty dictionary for the 'prior' feature\n",
    "# the value will be the 8 arabic marks (FATHA, DAMMA, KASRA, FATHATAN, DAMMATAN, KASRATAN, SUKUN, SHADDA) as a binary vector\n",
    "\n",
    "# then, loop over the tokenized input and check if the each character and word pair is not in the dictionary, get the indices of this word and its duplicates in the tokenized input array\n",
    "def get_prior(tokenized_input, gold_output):\n",
    "    prior = {}  # this dictionary will hold a key of tuple of 3 elements (word, character, index of character in the word) and the value will be the 8 arabic marks\n",
    "    for i in range(len(tokenized_input)):\n",
    "        if (tokenized_input[i], tokenized_input[i][0], 0) not in prior:\n",
    "            # get the indices of the word in the tokenized input array\n",
    "            indices = [j for j, x in enumerate(tokenized_input) if x == tokenized_input[i]]\n",
    "            # print(indices)\n",
    "            # get the words in the gold_output array with the same indices\n",
    "            words = []\n",
    "            maxi_len = 0\n",
    "            for j in indices:\n",
    "                if gold_output[j] not in words:\n",
    "                    words.append(gold_output[j])\n",
    "                    maxi_len = max(maxi_len, len(gold_output[j]))\n",
    "\n",
    "            for t in range(len(tokenized_input[i])):\n",
    "                prior[(tokenized_input[i], tokenized_input[i][t], t)] = [0, 0, 0, 0, 0, 0, 0, 0] # initialize the value of the key with zeros\n",
    "            \n",
    "            indx2 = 0\n",
    "            for word in words:\n",
    "                indx = 0\n",
    "                while indx < maxi_len:\n",
    "                    # extract the diacritics of word[indx]\n",
    "                    for iter in range(indx+1, len(word)):\n",
    "                        if is_not_arabic_diacritic(word[iter]):\n",
    "                            # print(iter)\n",
    "                            letter, tashkeel, shadda = araby.separate(word[indx: iter], extract_shadda=True) \n",
    "                            if diacritics_mapping['FATHA'] in tashkeel:         prior[(tokenized_input[i], word[indx], indx2)][0] = 1 \n",
    "                            if diacritics_mapping['DAMMA'] in tashkeel:         prior[(tokenized_input[i], word[indx], indx2)][1] = 1\n",
    "                            if diacritics_mapping['KASRA'] in tashkeel:         prior[(tokenized_input[i], word[indx], indx2)][2] = 1\n",
    "                            if diacritics_mapping['FATHATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], indx2)][3] = 1\n",
    "                            if diacritics_mapping['DAMMATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], indx2)][4] = 1\n",
    "                            if diacritics_mapping['KASRATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], indx2)][5] = 1\n",
    "                            if (diacritics_mapping['SUKUN'] in tashkeel and diacritics_mapping['SHADDA'] not in shadda):  \n",
    "                                prior[(tokenized_input[i], word[indx], indx2)][6] = 1 # if the letter has SHADDA, araby.separate() will return SUKUN in tashkeel and SHADDA in shadda, so to avoid this mislabeling we check if SHADDA not in shadda and if SUKUN in tashkeel, then this is a true SUKUN\n",
    "                            if diacritics_mapping['SHADDA'] in shadda:          prior[(tokenized_input[i], word[indx], indx2)][7] = 1\n",
    "                            indx = iter - 1\n",
    "                            indx2 += 1\n",
    "                            break \n",
    "                    indx += 1\n",
    "                indx2 = 0\n",
    "\n",
    "\n",
    "                indx = len(word) - 1    # my assumption is that the last character in the not a diacritic\n",
    "                if (not is_not_arabic_diacritic(word[len(word) - 1]) and is_not_arabic_diacritic(word[len(word) - 2])):  # if the last character is a diacritic and the one before it is not, then the index of the last character is len(word) - 2\n",
    "                    indx = len(word) - 2\n",
    "                elif (not is_not_arabic_diacritic(word[len(word) - 1]) and not is_not_arabic_diacritic(word[len(word) - 2])):  # if the last character is a diacritic and the one before it is also a diacritic (in shadda case), then the index of the last character is len(word) - 3\n",
    "                    indx = len(word) - 3\n",
    "\n",
    "\n",
    "                if (tokenized_input[i], word[indx], indx) not in prior:\n",
    "                    letter, tashkeel, shadda = araby.separate(word[indx: len(word)], extract_shadda=True) \n",
    "                    if diacritics_mapping['FATHA'] in tashkeel:         prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][0] = 1\n",
    "                    if diacritics_mapping['DAMMA'] in tashkeel:         prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][1] = 1\n",
    "                    if diacritics_mapping['KASRA'] in tashkeel:         prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][2] = 1 \n",
    "                    if diacritics_mapping['FATHATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][3] = 1 \n",
    "                    if diacritics_mapping['DAMMATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][4] = 1 \n",
    "                    if diacritics_mapping['KASRATAN'] in tashkeel:      prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][5] = 1\n",
    "                    if (diacritics_mapping['SUKUN'] in tashkeel and diacritics_mapping['SHADDA'] not in shadda):\n",
    "                        prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][6] = 1  # if the letter has SHADDA, araby.separate() will return SUKUN in tashkeel and SHADDA in shadda, so to avoid this mislabeling we check if SHADDA not in shadda and if SUKUN in tashkeel, then this is a true SUKUN\n",
    "                    if diacritics_mapping['SHADDA'] in shadda:          prior[(tokenized_input[i], word[indx], len(tokenized_input[i])-1)][7] = 1\n",
    "                    \n",
    "    return prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_tokenized_input = ['كإنكار', 'كإنكار', 'بقذر','بقذر', 'أكثر', 'أكثر', 'الزركشي']\n",
    "# test_gold_output = ['كَإِنْكَارِ','كَإِنْكَارٍ', 'بِقَذَر', 'بِقَذَرٍ','أكْثَرَ', 'أَكْثَرُ', 'الزَّرْكَشِيُّ']\n",
    "# print (get_prior(test_tokenized_input, test_gold_output))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # DON'T RUN THIS CODE AGAIN, THIS CELL TOOK 276 minutes TO RUN\n",
    "# # write in a file the prior feature\n",
    "# prior_feature = get_prior(tokenized_input, gold_output)\n",
    "# with open('./generatedFiles/training/prior_feature.txt', 'w', encoding='utf-8') as file:\n",
    "#     for key, value in prior_feature.items():\n",
    "#         file.write(f'{key}: {value}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_map(file_path, number_of_keys=2):\n",
    "    \"\"\"\n",
    "    Read the contents of the file located at file_path \n",
    "    and append to the dictionary prior_feature\n",
    "    \"\"\"\n",
    "    prior_feature = {}\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            key, value = line.strip().split(':')\n",
    "            key = key.strip()\n",
    "            value = value.strip()\n",
    "            key = key[1:-1].split(',')\n",
    "            value = value[1:-1].split(',')\n",
    "            if number_of_keys == 2:\n",
    "                key = (key[0][1:-1], key[1][2:-1], int(key[2]))\n",
    "            else:\n",
    "                key = (key[0][1:-1], key[1][2:-1], int(key[2]), int(key[3]))\n",
    "            \n",
    "            value = [int(i) for i in value]\n",
    "            prior_feature[key] = value\n",
    "    return prior_feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 0, 0, 0, 0, 0, 0, 0]\n"
     ]
    }
   ],
   "source": [
    "prior_feature = read_map('./generatedFiles/training/prior_feature.txt', 2)\n",
    "print(prior_feature[('قوله', 'ق', 0)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4 - CASE Feature: \n",
    "whether the letter expects a core word diacritic or a case ending. Case endings are placed on only one letter in a word, which may or may not be the last letter in the word. This is a binary feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from farasa.stemmer import FarasaStemmer\n",
    "\n",
    "# def arabic_stemmer(text):\n",
    "#     stemmer = FarasaStemmer(interactive=True)  # Set interactive to True for better performance\n",
    "\n",
    "#     # Perform stemming\n",
    "#     stemmed_text = stemmer.stem(text)\n",
    "\n",
    "#     return stemmed_text\n",
    "\n",
    "# # Example usage\n",
    "# input_text = \"الكتابة باللغة العربية\"\n",
    "# stemmed_text = arabic_stemmer(input_text)\n",
    "# print(\"Original text:\", input_text)\n",
    "# print(\"Stemmed text:\", stemmed_text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(10):\n",
    "#     stemmed_text = arabic_stemmer(tokenized_input[i])\n",
    "#     print(\"Original text:\", tokenized_input[i])\n",
    "#     print(\"Stemmed text:\", stemmed_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(len(tokenized_input)):\n",
    "#     stemmed_text = arabic_stemmer(tokenized_input[i])\n",
    "#     # Write and append on the tokenized input to a file\n",
    "#     with open('./generatedFiles/stemmed_input.txt', 'a', encoding='utf-8') as file:\n",
    "#         file.write(stemmed_text + '\\n')\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stemmed_text = stemmed_text.split(' ')\n",
    "# # write in a file the stemmed input\n",
    "# with open('./generatedFiles/stemmed_input.txt', 'w', encoding='utf-8') as file:\n",
    "#     for word in stemmed_text:\n",
    "#         file.write(word + '\\n')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5 - POS Tagging:\n",
    "Marking up a word in a text as corresponding to a particular part of speech, based on both its definition and its context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from farasa.pos import FarasaPOSTagger\n",
    "\n",
    "# tagger = FarasaPOSTagger(interactive=True)  # Download model if needed\n",
    "# text = \"قراءة يَحْتَاجُ الكتب مفيدة للعقل.\"\n",
    "# tagged = tagger.tag(text)\n",
    "# # Output: [['قراءة', 'NOUN'], ['الكتب', 'NOUN'], ['مفيدة', 'ADJ'], ['للعقل', 'NOUN'], ['.', 'PUNCT']]\n",
    "\n",
    "# print(tagged)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6 - Word2Vec\n",
    "Understanding context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sentences = LineSentence('./generatedFiles/training/new_input_sentence.txt')\n",
    "\n",
    "# Train Word2Vec model\n",
    "# model1 = FastText(sentences, vector_size=50, window=5, workers=4)\n",
    "# model1.save(\"word2vec.model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Get embeddings for two words\n",
    "# word1_embedding = model.wv[\"قال\"]\n",
    "# word2_embedding = model.wv[\"التفرغ\"]\n",
    "\n",
    "# print(word1_embedding)\n",
    "# print(word2_embedding)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Model Building**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_features_vector=[]\n",
    "tag_features_vector=[]\n",
    "prior_features_vector=[]\n",
    "embeddings = []\n",
    "\n",
    "for i in range(len(tokenized_input)):\n",
    "    for j in range(len(tokenized_input[i])):    \n",
    "        char_index = tokenizer_char.word_index.get(tokenized_input[i][j])\n",
    "        char_features_vector= char_embeddings[char_index]\n",
    "        if (len(tokenized_input[i]) != len(input_segments[i])):\n",
    "            input_segments[i] = \"S\" * (len(tokenized_input[i]) - len(input_segments[i])) + input_segments[i]\n",
    "        tag_index = tokenizer_tags.word_index.get(input_segments[i][j].lower())\n",
    "        tag_features_vector= tags_embeddings[tag_index]\n",
    "        prior_features_vector= prior_feature[(tokenized_input[i], tokenized_input[i][j], j)]\n",
    "        # pad the prior feature vector with zeros to have the same length as the other features\n",
    "        prior_features_vector = np.pad(prior_features_vector, (0, embedding_size-8), 'constant')\n",
    "        # concatenate the 3 features vectors to have a matrix of 3 columns\n",
    "        # embeddings.append(np.vstack((char_features_vector, tag_features_vector, prior_features_vector)))\n",
    "        # word_embedding = model.wv[tokenized_input[i]]\n",
    "        embeddings.append(np.concatenate((char_features_vector, tag_features_vector, prior_features_vector)))\n",
    "\n",
    "embeddings = np.array(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(char_features_vector)\n",
    "# print(tag_features_vector)\n",
    "# print(prior_features_vector)\n",
    "print(len(embeddings))\n",
    "print(embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the embeddings in a pickle file\n",
    "# with open('./generatedFiles/embeddings.pickle', 'wb') as file:\n",
    "#     pickle.dump(embeddings, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the embeddings from the pickle file\n",
    "# with open('./generatedFiles/embeddings.pickle', 'rb') as file:\n",
    "#     embeddings = pickle.load(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_map = {\n",
    "    (1, 0, 0, 0, 0, 0, 0, 0) : 0, # FATHA\n",
    "    (0, 0, 0, 1, 0, 0, 0, 0) : 1, # FATHATAN\n",
    "    (0, 0, 1, 0, 0, 0, 0, 0) : 2, # KASRA\n",
    "    (0, 0, 0, 0, 0, 1, 0, 0) : 3, # KASRATAN\n",
    "    (0, 1, 0, 0, 0, 0, 0, 0) : 4, # DAMMA\n",
    "    (0, 0, 0, 0, 1, 0, 0, 0) : 5, # DAMMATAN\n",
    "    (0, 0, 0, 0, 0, 0, 1, 0) : 6, # SUKUN\n",
    "    (0, 0, 0, 0, 0, 0, 0, 1) : 7,  # SHADDA\n",
    "    (1, 0, 0, 0, 0, 0, 0, 1) : 8, # SHADDA FATHA\n",
    "    (0, 0, 0, 1, 0, 0, 0, 1) : 9, # SHADDA FATHATAN\n",
    "    (0, 0, 1, 0, 0, 0, 0, 1) : 10, # SHADDA KASRA\n",
    "    (0, 0, 0, 0, 0, 1, 0, 1) : 11, # SHADDA KASRATAN\n",
    "    (0, 1, 0, 0, 0, 0, 0, 1) : 12, # SHADDA DAMMA\n",
    "    (0, 0, 0, 0, 1, 0, 0, 1) : 13, # SHADDA DAMMATAN\n",
    "    (0, 0, 0, 0, 0, 0, 0, 0) : 14\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./generatedFiles/training/gold_output_dict.txt', 'w', encoding='utf-8') as file:\n",
    "    for idx, word in enumerate(gold_output):\n",
    "        gold_diacritics = get_prior([tokenized_input[idx]], [word])\n",
    "        for key, value in gold_diacritics.items():\n",
    "            key = key + (idx,)\n",
    "            file.write(f'{key}: {value}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_output_dict = read_map('./generatedFiles/training/gold_output_dict.txt', 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_output_dict[('قوله', 'ق', 0, 0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change gold_output_dict.values() to a list of tuples\n",
    "for key, value in gold_output_dict.items():\n",
    "    gold_output_dict[key] = tuple(value)\n",
    "    \n",
    "gold_output_dict_values = list(gold_output_dict.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('./generatedFiles/training/gold_output_id.txt', 'w', encoding='utf-8') as file:\n",
    "#     for value in gold_output_dict_values:\n",
    "#         file.write(f'{output_map[value]}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the gold_output_id file\n",
    "gold_output_id = read_data('./generatedFiles/training/gold_output_id.txt')\n",
    "gold_output_id = np.array(gold_output_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(gold_output_id.shape)\n",
    "print(gold_output_id[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Length of embeddings = 8351478\n",
    "# Truncate emdeddings to have the 8353000\n",
    "embeddings_reshape = embeddings[:6000000]\n",
    "gold_output_id = gold_output_id[:6000000]\n",
    "\n",
    "# Make it np array \n",
    "embeddings_reshape = np.array(embeddings_reshape)\n",
    "gold_output_id = np.array(gold_output_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print (embeddings_reshape.shape)\n",
    "print (gold_output_id.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reshape embeddings to have 3 dimensions \n",
    "embeddings_reshape = embeddings_reshape.reshape((-1, 1000, 120))\n",
    "gold_output_id_reshape = gold_output_id.reshape(-1, 1000, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(embeddings_reshape.shape)\n",
    "print(gold_output_id_reshape.shape)\n",
    "\n",
    "# print the first 10 rows of the embeddings\n",
    "# print(embeddings[:10])\n",
    "\n",
    "# print the first 10 rows of the gold_output_id\n",
    "print(gold_output_id_reshape[:][0])\n",
    "\n",
    "# print the first 10 columns of the gold_output_id\n",
    "print(tf.keras.utils.to_categorical(gold_output_id_reshape[:][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Build a training model, first we need input layer that take matrix \"embeddings\" as an input with dropout of 10%\n",
    "# then we need a bidirectional LSTM layer with 100 units\n",
    "# then we need a dense layer with 100 units and relu activation function\n",
    "# then we need an output layer with 14 units and softmax activation function\n",
    "# use early stopping with patience of five epochs, a learning rate of 0.001, a batch size of 256, and an Adamax optimizer\n",
    "\n",
    "input_shape = (1000, 120)\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Bidirectional(LSTM(64, return_sequences=True), input_shape=input_shape))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Bidirectional(LSTM(64, return_sequences=True)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(Bidirectional(LSTM(64, return_sequences=True)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(TimeDistributed(Dense(128, activation='relu')))\n",
    "model.add(TimeDistributed(Dense(15, activation='softmax')))\n",
    "\n",
    "\n",
    "# compile the model\n",
    "model.compile(optimizer=Adamax(learning_rate=0.001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# summarize the model\n",
    "print(model.summary())\n",
    "\n",
    "# early stopping\n",
    "# early_stopping = EarlyStopping(monitor='test_accuracy', patience=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = []\n",
    "for i in range(len(gold_output_id_reshape)):\n",
    "    for j in range(1000):\n",
    "        labels.append( tf.keras.utils.to_categorical(gold_output_id_reshape[i][j], num_classes=15))\n",
    "        \n",
    "labels = np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = labels.reshape(-1, 1000, 15)\n",
    "print(labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fit the model on the training dataset and evaluate it on the validation dataset,\n",
    "# use early stopping with patience of five epochs, a learning rate of 0.001, a batch size of 256\n",
    "# before that, configure the model to use GPU\n",
    "\n",
    "# fit the model with gpu\n",
    "with tf.device('/GPU:0'):\n",
    "    model.fit(embeddings_reshape, labels, epochs=30, batch_size=256)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model\n",
    "model.save('./generatedFiles/model_shakkala.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Validation:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_data_before_preprocessing = read_data(\"./dataset/val.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(validation_data_before_preprocessing)):\n",
    "    save_tokenized_input(validation_data_before_preprocessing[i], path=\"./generatedFiles/validation/validation_tokenized_input.txt\", permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_tokenized_input = read_data(\"./generatedFiles/validation/validation_tokenized_input.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(validation_data_before_preprocessing)):\n",
    "    save_gold_output(validation_data_before_preprocessing[i],\"./generatedFiles/validation/validation_gold_output.txt\", permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_gold_output = read_data(\"./generatedFiles/validation/validation_gold_output.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(validation_tokenized_input)):\n",
    "    validation_segments, validation_seg_tags = get_seg_tags(validation_tokenized_input[i])\n",
    "    with open('./generatedFiles/validation/validation_input_segments.txt', 'a', encoding='utf-8') as file:\n",
    "        for tag in validation_seg_tags:\n",
    "            file.write(tag)\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_input_segments = read_data(\"./generatedFiles/validation/validation_input_segments.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_char_features_vector = []\n",
    "validation_tag_features_vector = []\n",
    "validation_prior_features_vector = []\n",
    "validation_embeddings = []\n",
    "\n",
    "for i in range(len(validation_tokenized_input)):\n",
    "    for j in range(len(validation_tokenized_input[i])):    \n",
    "        char_index = tokenizer_char.word_index.get(validation_tokenized_input[i][j])\n",
    "        validation_char_features_vector = char_embeddings[char_index]\n",
    "        if (len(validation_tokenized_input[i]) != len(validation_input_segments[i])):\n",
    "            validation_input_segments[i] = \"S\" * (len(validation_tokenized_input[i]) - len(validation_input_segments[i])) + validation_input_segments[i]\n",
    "        tag_index = tokenizer_tags.word_index.get(validation_input_segments[i][j].lower())\n",
    "        validation_tag_features_vector= tags_embeddings[tag_index]\n",
    "        validation_prior_features_vector= (prior_feature[(validation_tokenized_input[i], validation_tokenized_input[i][j], j)]) if (validation_tokenized_input[i], validation_tokenized_input[i][j], j) in prior_feature else [1, 1, 1, 1, 1, 1, 1, 1]\n",
    "        # pad the prior feature vector with zeros to have the same length as the other features\n",
    "        validation_prior_features_vector = np.pad(validation_prior_features_vector, (0, 32), 'constant')\n",
    "        # concatenate the 3 features vectors to have a matrix of 3 columns\n",
    "        # embeddings.append(np.vstack((char_features_vector, tag_features_vector, prior_features_vector)))\n",
    "        validation_embeddings.append(np.concatenate((validation_char_features_vector, validation_tag_features_vector, validation_prior_features_vector)))\n",
    "\n",
    "validation_embeddings = np.array(validation_embeddings)\n",
    "\n",
    "# print(char_features_vector)\n",
    "# print(tag_features_vector)\n",
    "# print(prior_features_vector)\n",
    "print(validation_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_embeddings_reshape = validation_embeddings[:421000]\n",
    "validation_embeddings_reshape = validation_embeddings_reshape.reshape((-1, 1000, 120))\n",
    "validation_embeddings_reshape = np.array(validation_embeddings_reshape)\n",
    "print(validation_embeddings_reshape.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gold labels\n",
    "with open('./generatedFiles/validation/validation_gold_output_dict.txt', 'w', encoding='utf-8') as file:\n",
    "    for idx, word in enumerate(validation_gold_output):\n",
    "        gold_diacritics = get_prior([validation_tokenized_input[idx]], [word])\n",
    "        for key, value in gold_diacritics.items():\n",
    "            key = key + (idx,)\n",
    "            file.write(f'{key}: {value}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_gold_output_dict = read_map('./generatedFiles/validation/validation_gold_output_dict.txt', 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change gold_output_dict.values() to a list of tuples\n",
    "for key, value in validation_gold_output_dict.items():\n",
    "    validation_gold_output_dict[key] = tuple(value)\n",
    "    \n",
    "validation_gold_output_dict_values = list(validation_gold_output_dict.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./generatedFiles/validation/validation_gold_output_id.txt', 'w', encoding='utf-8') as file:\n",
    "    for value in validation_gold_output_dict_values:\n",
    "        file.write(f'{output_map[value]}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_gold_output_id = read_data('./generatedFiles/validation/validation_gold_output_id.txt')\n",
    "validation_gold_output_id = np.array(validation_gold_output_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(validation_gold_output_id.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_gold_output_id = validation_gold_output_id[:421000]\n",
    "validation_gold_output_id = validation_gold_output_id.reshape(-1, 1000, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_labels = []\n",
    "for i in range(len(validation_gold_output_id)):\n",
    "    for j in range(1000):\n",
    "        validation_labels.append( tf.keras.utils.to_categorical(validation_gold_output_id[i][j], num_classes=15))\n",
    "        \n",
    "validation_labels = np.array(validation_labels)\n",
    "\n",
    "validation_labels = validation_labels.reshape(-1, 1000, 15)\n",
    "print(validation_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model = tf.keras.models.load_model('./generatedFiles/model_shakkala.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate the model\n",
    "\n",
    "loss, accuracy = model.evaluate(validation_embeddings_reshape, validation_labels, verbose=0)\n",
    "print('Accuracy: %f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data_before_preprocessing = read_data(\"./dataset/test.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_data_before_preprocessing)):\n",
    "    save_tokenized_input(test_data_before_preprocessing[i], path=\"./generatedFiles/test/test_tokenized_input.txt\", permission='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_tokenized_input = read_data(\"./generatedFiles/test/test_tokenized_input.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(test_tokenized_input)):\n",
    "    test_segments, test_seg_tags = get_seg_tags(test_tokenized_input[i])\n",
    "    with open('./generatedFiles/test/test_input_segments.txt', 'a', encoding='utf-8') as file:\n",
    "        for tag in test_seg_tags:\n",
    "            file.write(tag)\n",
    "        file.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_input_segments = read_data(\"./generatedFiles/test/test_input_segments.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(417359, 170)\n"
     ]
    }
   ],
   "source": [
    "test_char_features_vector=[]\n",
    "test_tag_features_vector=[]\n",
    "test_prior_features_vector=[]\n",
    "test_embeddings = []\n",
    "for i in range(len(test_tokenized_input)):\n",
    "    for j in range(len(test_tokenized_input[i])):    \n",
    "        char_index = tokenizer_char.word_index.get(test_tokenized_input[i][j])\n",
    "        test_char_features_vector= char_embeddings[char_index]\n",
    "        if (len(test_tokenized_input[i]) != len(test_input_segments[i])):\n",
    "            test_input_segments[i] = \"S\" * (len(test_tokenized_input[i]) - len(test_input_segments[i])) + test_input_segments[i]\n",
    "        tag_index = tokenizer_tags.word_index.get(test_input_segments[i][j].lower())\n",
    "        test_tag_features_vector= tags_embeddings[tag_index]\n",
    "        test_prior_features_vector= (prior_feature[(test_tokenized_input[i], test_tokenized_input[i][j], j)]) if (test_tokenized_input[i], test_tokenized_input[i][j], j) in prior_feature else [1, 1, 1, 1, 1, 1, 1, 1]\n",
    "        # pad the prior feature vector with zeros to have the same length as the other features\n",
    "        test_prior_features_vector = np.pad(test_prior_features_vector, (0, 32), 'constant')\n",
    "        # concatenate the 3 features vectors to have a matrix of 3 columns\n",
    "        # test_embeddings.append(np.vstack((test_char_features_vector, test_tag_features_vector, test_prior_features_vector)))\n",
    "        # word_embedding = model1.wv[test_tokenized_input[i]]\n",
    "        test_embeddings.append(np.concatenate((test_char_features_vector, test_tag_features_vector, test_prior_features_vector)))\n",
    "\n",
    "test_embeddings = np.array(test_embeddings)\n",
    "\n",
    "print(test_embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pad the embeddings to be of size 418000\n",
    "test_embeddings = np.pad(test_embeddings, ((0, 418000 - test_embeddings.shape[0]), (0, 0)), 'constant')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(418000, 170)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418, 1000, 170)\n"
     ]
    }
   ],
   "source": [
    "test_embeddings_reshape = test_embeddings[:418000]\n",
    "test_embeddings_reshape = test_embeddings_reshape.reshape((-1, 1000, 170))\n",
    "test_embeddings_reshape = np.array(test_embeddings_reshape)\n",
    "print(test_embeddings_reshape.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the model\n",
    "model = tf.keras.models.load_model('./generatedFiles/model_shakkala.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14/14 [==============================] - 8s 410ms/step\n"
     ]
    }
   ],
   "source": [
    "# predict the test dataset\n",
    "predictions = model.predict(test_embeddings_reshape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418, 1000, 15)\n"
     ]
    }
   ],
   "source": [
    "print(predictions.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = predictions.reshape(-1, 15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4.95688571e-03 6.73070531e-07 4.57599754e-06 4.32224078e-06\n",
      "  4.24501650e-06 1.12170767e-06 7.62303546e-03 8.01798701e-03\n",
      "  9.74423826e-01 2.39276287e-05 1.46317121e-03 4.60539413e-05\n",
      "  3.28106899e-03 5.04063719e-05 9.87033127e-05]\n",
      " [4.01821919e-03 1.61821354e-05 1.67270682e-05 1.83824552e-04\n",
      "  1.31599617e-03 4.30663567e-05 7.47109354e-01 1.33366091e-02\n",
      "  6.38496950e-02 4.72218890e-05 3.19705047e-02 4.36150614e-04\n",
      "  1.27261266e-01 7.16300390e-04 9.67886578e-03]\n",
      " [4.38582450e-01 4.64903351e-05 7.65887453e-05 1.51948771e-04\n",
      "  1.89074099e-01 1.93466505e-04 3.06609110e-03 1.60435890e-03\n",
      "  8.15489516e-03 1.97290974e-05 7.06350571e-03 2.82681081e-04\n",
      "  3.07692349e-01 4.31169290e-04 4.35602106e-02]\n",
      " [1.66252153e-06 1.27697968e-08 9.99973893e-01 1.06274257e-07\n",
      "  8.62993499e-09 5.22037213e-09 2.16680185e-09 1.33216123e-08\n",
      "  3.96678104e-08 1.90017890e-10 2.40693716e-05 3.76307234e-08\n",
      "  6.13984863e-09 1.34798519e-08 1.40929416e-07]\n",
      " [7.26842245e-06 1.26454552e-07 5.43859414e-07 1.24323080e-07\n",
      "  5.83269655e-09 4.31536442e-08 9.99985576e-01 1.40440211e-08\n",
      "  2.46091554e-06 2.11698392e-09 2.76890091e-06 1.41399141e-07\n",
      "  4.13616874e-08 5.18174431e-07 3.27426875e-07]\n",
      " [9.99998689e-01 5.35581535e-08 2.03363030e-07 8.09887268e-08\n",
      "  3.27069705e-09 1.96997405e-07 1.57188012e-07 1.84700912e-08\n",
      "  4.22926547e-08 8.25639823e-09 7.00323932e-09 2.96238056e-08\n",
      "  3.35826089e-10 2.15918323e-08 5.77606727e-07]\n",
      " [8.85960958e-07 6.12442408e-09 9.99997258e-01 6.28192595e-07\n",
      "  3.10748796e-08 1.96884926e-08 9.08758757e-09 1.72250181e-09\n",
      "  1.75476328e-10 4.85215867e-10 5.84484781e-07 1.07873994e-07\n",
      "  6.47902620e-10 6.55568932e-09 4.81436075e-07]\n",
      " [1.96899673e-05 3.07520495e-05 1.81955529e-05 3.82191320e-05\n",
      "  8.51408277e-06 2.15118998e-05 3.57698518e-05 1.06810643e-04\n",
      "  9.59753288e-06 1.10999950e-04 1.55026675e-04 7.51770567e-05\n",
      "  1.32387840e-05 3.91211106e-05 9.99317527e-01]\n",
      " [3.19606272e-07 2.95972200e-07 9.99461949e-01 2.74759122e-06\n",
      "  6.43753367e-07 2.19791659e-07 1.66805876e-07 5.88949547e-07\n",
      "  4.67449617e-08 3.44841418e-08 4.61505202e-04 1.77396964e-06\n",
      "  8.94051553e-08 3.70459219e-07 6.93517577e-05]\n",
      " [1.71817301e-04 8.04229694e-06 9.97229636e-01 1.22350320e-04\n",
      "  3.23302379e-06 1.07115284e-05 2.64121741e-06 4.33020796e-05\n",
      "  3.02573108e-06 5.65701203e-06 9.33816074e-04 1.01367092e-04\n",
      "  1.17114928e-06 9.82251731e-06 1.35346258e-03]]\n"
     ]
    }
   ],
   "source": [
    "print(predictions[:10])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the index of the maximum value in each row\n",
    "predictions = np.argmax(predictions, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(418000,)\n",
      "[ 8  6  0  2  6  0  2 14  2  2]\n"
     ]
    }
   ],
   "source": [
    "print(predictions.shape)\n",
    "print(predictions[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a dictionary to map the predictions to the corresponding unicode \n",
    "\n",
    "predictions_map = {\n",
    "    0 : diacritics_mapping['FATHA'],\n",
    "    1 : diacritics_mapping['FATHATAN'],\n",
    "    2 : diacritics_mapping['KASRA'],\n",
    "    3 : diacritics_mapping['KASRATAN'],\n",
    "    4 : diacritics_mapping['DAMMA'],\n",
    "    5 : diacritics_mapping['DAMMATAN'],\n",
    "    6 : diacritics_mapping['SUKUN'],\n",
    "    7 : diacritics_mapping['SHADDA'],\n",
    "    8 : diacritics_mapping['SHADDA'] + diacritics_mapping['FATHA'],\n",
    "    9 : diacritics_mapping['SHADDA'] + diacritics_mapping['FATHATAN'],\n",
    "    10 : diacritics_mapping['SHADDA'] + diacritics_mapping['KASRA'],\n",
    "    11 : diacritics_mapping['SHADDA'] + diacritics_mapping['KASRATAN'],\n",
    "    12 : diacritics_mapping['SHADDA'] + diacritics_mapping['DAMMA'],\n",
    "    13 : diacritics_mapping['SHADDA'] + diacritics_mapping['DAMMATAN'],\n",
    "    14 : ''\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# truncate the predictions to 417470\n",
    "predictions = predictions[:417470]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "417359\n",
      "['لَّ', 'يْ', 'سَ', 'لِ', 'لْ', 'وَ', 'كِ', 'ي', 'لِ', 'بِ']\n"
     ]
    }
   ],
   "source": [
    "# loop over the letters and concatenate it with the corresponding prediction\n",
    "predicted_diacritized_text = []\n",
    "count = 0\n",
    "for i in range(len(test_tokenized_input)):\n",
    "    for j in range(len(test_tokenized_input[i])):\n",
    "        predicted_diacritized_text.append(test_tokenized_input[i][j] + predictions_map[predictions[count]])\n",
    "        count += 1\n",
    "        \n",
    "print(len(predicted_diacritized_text))\n",
    "print(predicted_diacritized_text[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./generatedFiles/test/predictions.csv', 'w', encoding='utf-8') as file:\n",
    "    file.write('ID,label\\n')\n",
    "    id = 0\n",
    "    id2 = 0\n",
    "    for i in range(len(test_tokenized_input)):\n",
    "        for j in range(len(test_tokenized_input[i])):\n",
    "            # check if test_tokenized_input[i][j] is an arabic letter\n",
    "            if test_tokenized_input[i][j] in arabic_letters:\n",
    "                file.write(f'{id2},{predictions[id]}\\n')\n",
    "                id2 += 1\n",
    "            id += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "x ={\n",
    "    0 : 0,\n",
    "    1 : 1,\n",
    "    2 : 4,\n",
    "    3 : 5,\n",
    "    4 : 2,\n",
    "    5 : 3,\n",
    "    6 : 6,\n",
    "    7 : 7,\n",
    "    8 : 8,\n",
    "    9 : 9,\n",
    "    10 : 12,\n",
    "    11 : 13,\n",
    "    12 : 10,\n",
    "    13 : 11,\n",
    "    14 : 14\n",
    "    \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read csv file, and loop over the csv, for the second column replace each value with its corresponding value in x\n",
    "import csv\n",
    "data = []\n",
    "with open('./generatedFiles/test/predictions.csv', 'r') as file:\n",
    "    reader = csv.reader(file)\n",
    "    for row in reader:\n",
    "        data.append(row)\n",
    "        \n",
    "for i in range(1, len(data)):\n",
    "    data[i][1] = x[int(data[i][1])]\n",
    "    \n",
    "    \n",
    "    \n",
    "# write the new data to the csv file\n",
    "with open(\"./generatedFiles/test/predictions_updated.csv\", mode='w', newline='') as file:\n",
    "    writer = csv.writer(file)\n",
    "    writer.writerows(data)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}