normalize.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "from itertools import chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "songs = pd.read_pickle('all_songs.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "harakatRe=r'(ٍ|َ|ُ|ِ|ّ|ْ|ً)' # to delete\n",
    "lifRe=r'(آ|أ|إ|آ)'              # to normalize\n",
    "hahRe=r'ه\\Z'                    # to normalize\n",
    "yahRe=r'ى$'                     # to normalize\n",
    "spaceRe=r'\\n$'                     # to normalize\n",
    "waRe = r'^و'                    # to delete\n",
    "yaRe = r'^يا'                    # to delete\n",
    "indRe = u'\t'                    # to delete\n",
    "madRe = u'ـ'                    # to delete\n",
    "starRe = r'[*]?'                    # to delete\n",
    "repeatRe = r'(.)\\1*'            # to delete\n",
    "spRe= u'\\xa0'                    # to delete\n",
    "tabRe= u'\\t'                    # to delete\n",
    "arRe= u'\\r'                    # to delete\n",
    "dashRe= u'-'                    # to delete\n",
    "engLettersRe= r'[A-Za-z0-9]'                    # to delete\n",
    "noncharsRe = r'[=|#|;|(|)|{|}|؟|&|:|,|!|_|\\'|\\\\|/]'\n",
    "\n",
    "def normalize_words(tokens):\n",
    "    words = []\n",
    "    for word in tokens:\n",
    "        word = re.sub(harakatRe,u'',word,flags=re.U)\n",
    "        word = re.sub(lifRe,u'ا',word,flags=re.U)\n",
    "        word = re.sub(hahRe,u'ه',word,flags=re.U)\n",
    "        word = re.sub(yahRe,u'ي',word,flags=re.U)\n",
    "        word = re.sub(waRe,u'و ',word,flags=re.U)\n",
    "        word = re.sub(yaRe,u'يا ',word,flags=re.U)\n",
    "        word = re.sub(madRe,u'',word,flags=re.U)\n",
    "        word = re.sub(indRe,u'',word,flags=re.U)\n",
    "        word = re.sub(starRe,u'',word,flags=re.U)\n",
    "        word = re.sub(r'(\\n){1,}', '\\n', word,flags=re.U)\n",
    "        word = re.sub(u'[.]', u'', word,flags=re.U)\n",
    "        word = re.sub(r'[Â|Ã|é|à|ƒ]', u'', word,flags=re.U)\n",
    "        word = re.sub(spRe, u'', word,flags=re.U)\n",
    "        word = re.sub(tabRe, u'', word,flags=re.U)\n",
    "        word = re.sub(arRe, u'', word,flags=re.U)\n",
    "        word = re.sub(dashRe, u'', word,flags=re.U)\n",
    "        word = re.sub(engLettersRe, u'', word,flags=re.U)\n",
    "        word = re.sub(noncharsRe, u'', word,flags=re.U)\n",
    "        word = re.sub(r'(.)\\1+',r'\\1', word, flags=re.U)\n",
    "        \n",
    "        word = word.split(' ')\n",
    "        words.append(word)\n",
    "    return list(chain.from_iterable(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 58.9 s, sys: 52 ms, total: 59 s\n",
      "Wall time: 59 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "with open('aggregate_lyrics.txt', 'w') as f: \n",
    "    for i in range(songs.shape[0]):\n",
    "        song = songs.lyrics[i]\n",
    "        tokens = re.sub(r'(w)\\1*','',song,re.U).replace('\\n', '\\n ').replace('\\n', ' \\n').strip().split(r' ')\n",
    "        normalized_lyrics = re.sub(r'\\n(\\s{2,})', '\\n',' '.join(normalize_words(tokens))) #' '.join(normalize_words(tokens))\n",
    "        normalized_lyrics = ''.join(normalized_lyrics+'\\n')\n",
    "        f.write(normalized_lyrics) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "        #tokens = re.sub(r'\\r',' ',song.replace('\\n', ' \\n '),re.U).strip().split(r' ')\n",
    "        #tokens = re.sub(r'(w)\\1*','_',song,re.U).replace('\\n', '\\n ').replace('\\n', ' \\n').strip().split(r' ')\n",
    "        #normalized_lyrics = ' '.join(normalize_words(tokens).strip().replace('\\n  ', ''))\n",
    "        #normalized_lyrics = ''.join(normalized_lyrics+'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}