train_vectors_mixing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import numpy as np\n",
    "from gensim.models import word2vec\n",
    "import io, re\n",
    "from gensim.models import Word2Vec\n",
    "from gensim.models.word2vec import LineSentence\n",
    "import logging\n",
    " \n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
    "\n",
    "def valid_and_separate(line):\n",
    "    vals = line.split(';')\n",
    "    if len(vals) != 2:\n",
    "        return None\n",
    "    for i in range(2):\n",
    "        v = re.sub(r'[\\.\\,\\%«\"\\)\\(]', '', vals[i]).strip()\n",
    "        if not v:\n",
    "            return None\n",
    "        vals[i] = v\n",
    "    if vals[0] == vals[1]:\n",
    "        return None\n",
    "    return vals\n",
    "\n",
    "def readvocub(filename):\n",
    "    tr = {}\n",
    "    with io.open(filename, 'r', encoding='utf8') as fin:\n",
    "        for line in fin:\n",
    "            vals = valid_and_separate(line)\n",
    "            if vals is None:\n",
    "                continue\n",
    "            tr[vals[0]] = vals[1]\n",
    "    fin.close()\n",
    "    return tr\n",
    "\n",
    "def mix_text(target_file, source_file, dictionary, translate_limit, window_limit,verbose=1):\n",
    "    words_count = 0\n",
    "    translated_count = 0\n",
    "    order_count = 0\n",
    "    for line in source_file:\n",
    "        if(words_count % 1000 == 0 and verbose):\n",
    "            print \"\\r processed words: {0}K, translated words: {1}K\".format(words_count/1000,translated_count/1000),\n",
    "        if(words_count < translate_limit):\n",
    "            words = line.split()\n",
    "            words_count += len(words)\n",
    "            for word in words:\n",
    "                w = None\n",
    "                if(word in dictionary and order_count > window_limit):\n",
    "                    w = dictionary[word].lower()\n",
    "                    order_count = 0\n",
    "                    translated_count+=1\n",
    "                else:\n",
    "                    order_count += 1\n",
    "                    w = word\n",
    "                target_file.write(w + ' ')\n",
    "            target_file.write(u'\\r\\n')\n",
    "        else:\n",
    "            words_count += len(line.split())\n",
    "            target_file.write(line)\n",
    "            target_file.write(u'\\r\\n')\n",
    "    print ' '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start eng\n",
      " processed words: 267186K, translated words: 13010K  \n",
      "Start rus\n",
      " processed words: 197433K, translated words: 9405K  \n"
     ]
    }
   ],
   "source": [
    "mix = True\n",
    "file_corpora = 'data\\\\rus_en_mixed_file.txt'\n",
    "if mix:\n",
    "    # load word pairs\n",
    "    en_ru = readvocub('data\\\\eng_rus_vocab.txt')\n",
    "    ru_en = readvocub('data\\\\rus_eng_vocab.txt')\n",
    "    with io.open(file_corpora, 'w', encoding='utf8') as out_file:\n",
    "        with io.open('data\\\\eng_corpora.txt','r', encoding='utf8') as eng_f:\n",
    "            print 'Start eng'\n",
    "            mix_text(out_file,eng_f,en_ru,11e+7,6)\n",
    "        with io.open('data\\\\rus_corpora.txt','r', encoding='utf8') as rus_f:\n",
    "            print 'Start rus'\n",
    "            mix_text(out_file,rus_f,ru_en,8e+7,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train W2V for target\n"
     ]
    }
   ],
   "source": [
    "load_from_file = False\n",
    "model_save = 'data\\\\bilingual_mixed'\n",
    "if load_from_file:\n",
    "    model = Word2Vec.load(model_save)\n",
    "else:\n",
    "    print 'Train W2V for target'\n",
    "    sentences = LineSentence(file_corpora)\n",
    "    model = Word2Vec(sentences, size=100, window=6, min_count=4, sg=1, workers=6)\n",
    "    model.save(model_save)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}