eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm as log_progress\n",
    "\n",
    "from itertools import chain\n",
    "from razdel.utils import (\n",
    "    join_path,\n",
    "    list_paths,\n",
    "    load_lines,\n",
    "    dump_lines,\n",
    "    load_xml\n",
    ")\n",
    "from razdel.eval.etl.syntag import (\n",
    "    parse_sents as parse_syntag_sents,\n",
    "    parse_tokens as parse_syntag_tokens\n",
    ")\n",
    "from razdel.eval.etl.corpora import (\n",
    "    parse_sents as parse_corpora_sents,\n",
    "    parse_tokens as parse_corpora_tokens\n",
    ")\n",
    "from razdel.eval.etl.gicrya import (\n",
    "    parse_sents as parse_gicrya_sents,\n",
    "    parse_tokens as parse_gicrya_tokens\n",
    ")\n",
    "from razdel.eval.etl.rnc import (\n",
    "    parse_sents as parse_rnc_sents,\n",
    "    parse_tokens as parse_rnc_tokens\n",
    ")\n",
    "from razdel.tests.partition import (\n",
    "    format_partitions,\n",
    "    parse_partitions\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Syntag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !mkdir -p data/raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/UniversalDependencies/UD_Russian-SynTagRus/archive/master.zip -P data/raw\n",
    "# !unzip data/raw/master.zip -d data/raw\n",
    "# !rm data/raw/master.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paths = list_paths(join_path('data', 'raw', 'UD_Russian-SynTagRus-master', '*.conllu'))\n",
    "# lines = chain.from_iterable(load_lines(_) for _ in paths)\n",
    "# partitions = parse_syntag_sents(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'syntag_sents.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# paths = list_paths(join_path('data', 'raw', 'UD_Russian-SynTagRus-master', '*.conllu'))\n",
    "# lines = chain.from_iterable(load_lines(_) for _ in paths)\n",
    "# partitions = parse_syntag_tokens(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'syntag_tokens.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget http://opencorpora.org/files/export/annot/annot.opcorpora.xml.bz2 -P data/raw\n",
    "# !bunzip2 -d data/raw/annot.opcorpora.xml.bz2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stream = load_xml(join_path('data', 'raw', 'annot.opcorpora.xml'))\n",
    "# records = parse_corpora_sents(stream)\n",
    "# records = log_progress(records)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'corpora_sents.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stream = load_xml(join_path('data', 'raw', 'annot.opcorpora.xml'))\n",
    "# partitions = parse_corpora_tokens(stream)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'corpora_tokens.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gicrya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/dialogue-evaluation/morphoRuEval-2017/raw/master/GICRYA_texts.zip -P data/raw\n",
    "# !unzip data/raw/GICRYA_texts.zip -d data/raw\n",
    "# !rm data/raw/GICRYA_texts.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lines = load_lines(join_path('data', 'raw', 'gikrya_fixed.txt'))\n",
    "# partitions = parse_gicrya_sents(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'gicrya_sents.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lines = load_lines(join_path('data', 'raw', 'gikrya_fixed.txt'))\n",
    "# partitions = parse_gicrya_tokens(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'gicrya_tokens.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RNC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://github.com/dialogue-evaluation/morphoRuEval-2017/raw/master/RNC_texts.rar -P data/raw\n",
    "# !unrar x data/raw/RNC_texts.rar \n",
    "# !mv RNCgoldInUD_Morpho.conll data/raw\n",
    "# !rm data/raw/RNC_texts.rar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lines = load_lines(join_path('data', 'raw', 'RNCgoldInUD_Morpho.conll'))\n",
    "# partitions = parse_rnc_sents(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'rnc_sents.txt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lines = load_lines(join_path('data', 'raw', 'RNCgoldInUD_Morpho.conll'))\n",
    "# partitions = parse_rnc_tokens(lines)\n",
    "# partitions = log_progress(partitions)\n",
    "# lines = format_partitions(partitions)\n",
    "# dump_lines(lines, join_path('data', 'rnc_tokens.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from razdel.utils import (\n",
    "    join_path,\n",
    "    load_lines,\n",
    ")\n",
    "from razdel.tests.partition import parse_partitions\n",
    "\n",
    "\n",
    "lines = load_lines(join_path('data', 'syntag_sents.txt'))\n",
    "syntag_sents = list(parse_partitions(lines))\n",
    "\n",
    "lines = load_lines(join_path('data', 'corpora_sents.txt'))\n",
    "corpora_sents = list(parse_partitions(lines))\n",
    "\n",
    "lines = load_lines(join_path('data', 'gicrya_sents.txt'))\n",
    "gicrya_sents = list(parse_partitions(lines))\n",
    "\n",
    "lines = load_lines(join_path('data', 'rnc_sents.txt'))\n",
    "rnc_sents = list(parse_partitions(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from razdel.eval.zoo import dot_sentenize\n",
    "\n",
    "# pip install rusenttokenize\n",
    "from razdel.eval.zoo import deepmipt_sentenize\n",
    "\n",
    "# pip install nltk\n",
    "# nltk.download('punkt')\n",
    "# wget https://raw.githubusercontent.com/mhq/train_punkt/master/russian.pickle\n",
    "#      -O ~/nltk_data/tokenizers/punkt/PY3/russian.pickle\n",
    "from razdel.eval.zoo import nltk_sentenize\n",
    "\n",
    "# pip install segtok\n",
    "from razdel.eval.zoo import segtok_sentenize\n",
    "\n",
    "# pip install mosestokenizer\n",
    "from razdel.eval.zoo import moses_sentenize\n",
    "\n",
    "from razdel import sentenize\n",
    "\n",
    "\n",
    "# Texterra\n",
    "# Можно ещё сравнивать с https://texterra.ispras.ru/products, но\n",
    "# 1. она медленно работает, как минимум затраты на http\n",
    "# 2. иногда кидает ошибку (возможно дело в английских предложениях)\n",
    "# 3. качество немного выше segtok\n",
    "\n",
    "\n",
    "# Polyglot\n",
    "# реализует http://www.unicode.org/reports/tr29/\n",
    "\n",
    "# Сорян, не смог установить. Дикие траблы с ICU\n",
    "# brew install icu4c\n",
    "# export ICU_VERSION=62.1\n",
    "# export BASE=/usr/local/Cellar/icu4c/\n",
    "# export PATH=$PATH:$BASE/$ICU_VERSION/bin\n",
    "# export PYICU_INCLUDES=$BASE/$ICU_VERSION/include\n",
    "# export PYICU_LFLAGS=-L$BASE/$ICU_VERSION/lib\n",
    "# pip install pyicu polyglot\n",
    "\n",
    "# Вроде установилось но \n",
    "# > from polyglot.text import Text\n",
    "# > Text('...')\n",
    "# Symbol not found: __ZNK6icu_6214Transliterator12getTargetSetERNS_10UnicodeSetE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from tqdm import tqdm as log_progress\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "from razdel.eval.tests import (\n",
    "    generate_precision_tests,\n",
    "    generate_recall_tests,\n",
    "    correct_precision,\n",
    "    correct_recall\n",
    ")\n",
    "from razdel.eval.report import (\n",
    "    Dataset,\n",
    "    Model,\n",
    "    Metric,\n",
    "    report_tasks,\n",
    "    run_task\n",
    ")\n",
    "\n",
    "datasets = [\n",
    "    Dataset('corpora', corpora_sents),\n",
    "    Dataset('syntag', syntag_sents),\n",
    "    Dataset('gicrya', gicrya_sents),\n",
    "    Dataset('rnc', rnc_sents),\n",
    "]\n",
    "models = [\n",
    "    Model('re.split([.?!…])', dot_sentenize),\n",
    "    Model('moses', moses_sentenize),\n",
    "    Model('nltk.sent_tokenize', nltk_sentenize),\n",
    "    Model('segtok.split_single', segtok_sentenize),\n",
    "    Model('deepmipt', deepmipt_sentenize),\n",
    "    Model('razdel.sentenize', sentenize),\n",
    "]\n",
    "metrics = [\n",
    "    Metric('precision', generate_precision_tests, correct_precision),\n",
    "    Metric('recall', generate_recall_tests, correct_recall)\n",
    "]\n",
    "\n",
    "\n",
    "tasks = list(report_tasks(datasets, models, metrics))\n",
    "results = Parallel(n_jobs=4)(\n",
    "    delayed(run_task)(_)\n",
    "    for _ in log_progress(tasks)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from razdel.eval.report import (\n",
    "    result_rows,\n",
    "    show_results\n",
    ")\n",
    "\n",
    "\n",
    "table = pd.DataFrame(result_rows(results))\n",
    "print(show_results(table).to_html())\n",
    "show_results(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table border=\"1\" class=\"dataframe\">\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th></th>\n",
    "      <th colspan=\"4\" halign=\"left\">corpora</th>\n",
    "      <th colspan=\"4\" halign=\"left\">syntag</th>\n",
    "      <th colspan=\"4\" halign=\"left\">gicrya</th>\n",
    "      <th colspan=\"4\" halign=\"left\">rnc</th>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th></th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <th>re.split([.?!…])</th>\n",
    "      <td>8027</td>\n",
    "      <td>5615</td>\n",
    "      <td>2412</td>\n",
    "      <td>10.21</td>\n",
    "      <td>2188</td>\n",
    "      <td>1761</td>\n",
    "      <td>427</td>\n",
    "      <td>6.63</td>\n",
    "      <td>4096</td>\n",
    "      <td>2413</td>\n",
    "      <td>1683</td>\n",
    "      <td>7.97</td>\n",
    "      <td>8191</td>\n",
    "      <td>6587</td>\n",
    "      <td>1604</td>\n",
    "      <td>9.93</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>moses</th>\n",
    "      <td>17131</td>\n",
    "      <td>8862</td>\n",
    "      <td>8269</td>\n",
    "      <td>70.89</td>\n",
    "      <td>8274</td>\n",
    "      <td>5255</td>\n",
    "      <td>3019</td>\n",
    "      <td>46.53</td>\n",
    "      <td>6698</td>\n",
    "      <td>1800</td>\n",
    "      <td>4898</td>\n",
    "      <td>55.81</td>\n",
    "      <td>21743</td>\n",
    "      <td>9300</td>\n",
    "      <td>12443</td>\n",
    "      <td>69.11</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>nltk.sent_tokenize</th>\n",
    "      <td>7752</td>\n",
    "      <td>3703</td>\n",
    "      <td>4049</td>\n",
    "      <td>56.19</td>\n",
    "      <td>1907</td>\n",
    "      <td>951</td>\n",
    "      <td>956</td>\n",
    "      <td>34.45</td>\n",
    "      <td>2212</td>\n",
    "      <td>829</td>\n",
    "      <td>1383</td>\n",
    "      <td>37.63</td>\n",
    "      <td>11390</td>\n",
    "      <td>6216</td>\n",
    "      <td>5174</td>\n",
    "      <td>48.21</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>segtok.split_single</th>\n",
    "      <td>8981</td>\n",
    "      <td>2306</td>\n",
    "      <td>6675</td>\n",
    "      <td>78.10</td>\n",
    "      <td>1971</td>\n",
    "      <td>1090</td>\n",
    "      <td>881</td>\n",
    "      <td>49.53</td>\n",
    "      <td>79135</td>\n",
    "      <td>914</td>\n",
    "      <td>78221</td>\n",
    "      <td>14.41</td>\n",
    "      <td>86252</td>\n",
    "      <td>1206</td>\n",
    "      <td>85046</td>\n",
    "      <td>22.83</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>deepmipt</th>\n",
    "      <td>4529</td>\n",
    "      <td>925</td>\n",
    "      <td>3604</td>\n",
    "      <td>37.55</td>\n",
    "      <td>579</td>\n",
    "      <td>114</td>\n",
    "      <td>465</td>\n",
    "      <td>23.20</td>\n",
    "      <td>3502</td>\n",
    "      <td>1647</td>\n",
    "      <td>1855</td>\n",
    "      <td>26.86</td>\n",
    "      <td>7487</td>\n",
    "      <td>5965</td>\n",
    "      <td>1522</td>\n",
    "      <td>25.74</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>razdel.sentenize</th>\n",
    "      <td>4323</td>\n",
    "      <td>584</td>\n",
    "      <td>3739</td>\n",
    "      <td>26.79</td>\n",
    "      <td>369</td>\n",
    "      <td>92</td>\n",
    "      <td>277</td>\n",
    "      <td>16.51</td>\n",
    "      <td>5872</td>\n",
    "      <td>1276</td>\n",
    "      <td>4596</td>\n",
    "      <td>19.25</td>\n",
    "      <td>4903</td>\n",
    "      <td>2007</td>\n",
    "      <td>2896</td>\n",
    "      <td>19.65</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(show_results(table, github=True).to_html(escape=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm as log_progress\n",
    "\n",
    "from razdel.utils import (\n",
    "    join_path,\n",
    "    load_lines,\n",
    ")\n",
    "from razdel.tests.partition import parse_partitions\n",
    "\n",
    "\n",
    "lines = log_progress(load_lines(join_path('data', 'syntag_tokens.txt')))\n",
    "syntag_tokens = list(parse_partitions(lines))\n",
    "\n",
    "lines = log_progress(load_lines(join_path('data', 'corpora_tokens.txt')))\n",
    "corpora_tokens = list(parse_partitions(lines))\n",
    "\n",
    "lines = log_progress(load_lines(join_path('data', 'gicrya_tokens.txt')))\n",
    "gicrya_tokens = list(parse_partitions(lines))\n",
    "\n",
    "lines = log_progress(load_lines(join_path('data', 'rnc_tokens.txt')))\n",
    "rnc_tokens = list(parse_partitions(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from razdel.eval.zoo import (\n",
    "    re_tokenize,\n",
    "    nltk_tokenize,  # see nltk_sentenize\n",
    ")\n",
    "\n",
    "# pip install spacy\n",
    "from razdel.eval.zoo import spacy_tokenize\n",
    "\n",
    "# pip install git+https://github.com/aatimofeev/spacy_russian_tokenizer.git\n",
    "from razdel.eval.zoo import spacy_tokenize2\n",
    "\n",
    "# pip install pymystem3\n",
    "from razdel.eval.zoo import mystem_tokenize\n",
    "\n",
    "#pip install segtok\n",
    "from razdel.eval.zoo import segtok_tokenize\n",
    "\n",
    "#pip install mosestokenizer\n",
    "from razdel.eval.zoo import moses_tokenize\n",
    "\n",
    "from razdel import tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "from random import seed, sample\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "from razdel.eval.tests import (\n",
    "    generate_precision_tests,\n",
    "    generate_recall_tests,\n",
    "    correct_precision,\n",
    "    correct_recall\n",
    ")\n",
    "from razdel.eval.report import (\n",
    "    Dataset,\n",
    "    Model,\n",
    "    Metric,\n",
    "    report_tasks,\n",
    "    run_task\n",
    ")\n",
    "\n",
    "\n",
    "def sample_(items):\n",
    "    seed(1)\n",
    "    return sample(items, 10000)\n",
    "\n",
    "\n",
    "datasets = [\n",
    "    Dataset('corpora', sample_(corpora_tokens)),\n",
    "    Dataset('syntag', sample_(syntag_tokens)),\n",
    "    Dataset('gicrya', sample_(gicrya_tokens)),\n",
    "    Dataset('rnc', sample_(rnc_tokens)),\n",
    "]\n",
    "models = [\n",
    "    Model('re.findall(\\w+|\\d+|\\p+)', re_tokenize),\n",
    "    Model('nltk.word_tokenize', nltk_tokenize),\n",
    "    Model('spacy_tokenize', spacy_tokenize),\n",
    "    Model('spacy_tokenize2', spacy_tokenize2),\n",
    "    Model('mystem', mystem_tokenize),\n",
    "    Model('moses', moses_tokenize),\n",
    "    Model('segtok.word_tokenize', segtok_tokenize),\n",
    "    Model('razdel.tokenize', tokenize),\n",
    "]\n",
    "metrics = [\n",
    "    Metric('precision', generate_precision_tests, correct_precision),\n",
    "    Metric('recall', generate_recall_tests, correct_recall)\n",
    "]\n",
    "\n",
    "\n",
    "tasks = list(report_tasks(datasets, models, metrics))\n",
    "results = Parallel(n_jobs=4)(\n",
    "    delayed(run_task)(_)\n",
    "    for _ in log_progress(tasks)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "from razdel.eval.report import (\n",
    "    result_rows,\n",
    "    show_results\n",
    ")\n",
    "\n",
    "table = pd.DataFrame(result_rows(results))\n",
    "print(show_results(table).to_html())\n",
    "show_results(table)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table border=\"1\" class=\"dataframe\">\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th></th>\n",
    "      <th colspan=\"4\" halign=\"left\">corpora</th>\n",
    "      <th colspan=\"4\" halign=\"left\">syntag</th>\n",
    "      <th colspan=\"4\" halign=\"left\">gicrya</th>\n",
    "      <th colspan=\"4\" halign=\"left\">rnc</th>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th></th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "      <th>errors</th>\n",
    "      <th>prec</th>\n",
    "      <th>recall</th>\n",
    "      <th>time</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <th>re.findall(\\w+|\\d+|\\p+)</th>\n",
    "      <td>1863</td>\n",
    "      <td>1857</td>\n",
    "      <td>6</td>\n",
    "      <td>13.02</td>\n",
    "      <td>1613</td>\n",
    "      <td>1460</td>\n",
    "      <td>153</td>\n",
    "      <td>14.79</td>\n",
    "      <td>1188</td>\n",
    "      <td>1188</td>\n",
    "      <td>0</td>\n",
    "      <td>11.18</td>\n",
    "      <td>5005</td>\n",
    "      <td>5002</td>\n",
    "      <td>3</td>\n",
    "      <td>12.71</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>nltk.word_tokenize</th>\n",
    "      <td>1666</td>\n",
    "      <td>120</td>\n",
    "      <td>1546</td>\n",
    "      <td>142.75</td>\n",
    "      <td>1685</td>\n",
    "      <td>313</td>\n",
    "      <td>1372</td>\n",
    "      <td>141.00</td>\n",
    "      <td>169</td>\n",
    "      <td>169</td>\n",
    "      <td>0</td>\n",
    "      <td>102.86</td>\n",
    "      <td>1987</td>\n",
    "      <td>1752</td>\n",
    "      <td>235</td>\n",
    "      <td>118.40</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>spacy_tokenize</th>\n",
    "      <td>2846</td>\n",
    "      <td>1122</td>\n",
    "      <td>1724</td>\n",
    "      <td>75.87</td>\n",
    "      <td>2068</td>\n",
    "      <td>987</td>\n",
    "      <td>1081</td>\n",
    "      <td>72.71</td>\n",
    "      <td>905</td>\n",
    "      <td>905</td>\n",
    "      <td>0</td>\n",
    "      <td>50.54</td>\n",
    "      <td>2706</td>\n",
    "      <td>2706</td>\n",
    "      <td>0</td>\n",
    "      <td>56.91</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>spacy_tokenize2</th>\n",
    "      <td>2102</td>\n",
    "      <td>378</td>\n",
    "      <td>1724</td>\n",
    "      <td>94.97</td>\n",
    "      <td>1272</td>\n",
    "      <td>191</td>\n",
    "      <td>1081</td>\n",
    "      <td>89.86</td>\n",
    "      <td>226</td>\n",
    "      <td>226</td>\n",
    "      <td>0</td>\n",
    "      <td>65.23</td>\n",
    "      <td>1877</td>\n",
    "      <td>1877</td>\n",
    "      <td>0</td>\n",
    "      <td>71.46</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>mystem</th>\n",
    "      <td>2577</td>\n",
    "      <td>908</td>\n",
    "      <td>1669</td>\n",
    "      <td>74.31</td>\n",
    "      <td>2137</td>\n",
    "      <td>720</td>\n",
    "      <td>1417</td>\n",
    "      <td>73.27</td>\n",
    "      <td>1156</td>\n",
    "      <td>950</td>\n",
    "      <td>206</td>\n",
    "      <td>58.63</td>\n",
    "      <td>1297</td>\n",
    "      <td>1124</td>\n",
    "      <td>173</td>\n",
    "      <td>62.89</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>moses</th>\n",
    "      <td>1335</td>\n",
    "      <td>187</td>\n",
    "      <td>1148</td>\n",
    "      <td>65.89</td>\n",
    "      <td>1405</td>\n",
    "      <td>338</td>\n",
    "      <td>1067</td>\n",
    "      <td>64.89</td>\n",
    "      <td>808</td>\n",
    "      <td>808</td>\n",
    "      <td>0</td>\n",
    "      <td>57.03</td>\n",
    "      <td>1748</td>\n",
    "      <td>1745</td>\n",
    "      <td>3</td>\n",
    "      <td>56.51</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>segtok.word_tokenize</th>\n",
    "      <td>414</td>\n",
    "      <td>122</td>\n",
    "      <td>292</td>\n",
    "      <td>53.84</td>\n",
    "      <td>584</td>\n",
    "      <td>334</td>\n",
    "      <td>250</td>\n",
    "      <td>53.72</td>\n",
    "      <td>830</td>\n",
    "      <td>830</td>\n",
    "      <td>0</td>\n",
    "      <td>52.38</td>\n",
    "      <td>1252</td>\n",
    "      <td>1249</td>\n",
    "      <td>3</td>\n",
    "      <td>43.12</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <th>razdel.tokenize</th>\n",
    "      <td>348</td>\n",
    "      <td>270</td>\n",
    "      <td>78</td>\n",
    "      <td>28.96</td>\n",
    "      <td>460</td>\n",
    "      <td>383</td>\n",
    "      <td>77</td>\n",
    "      <td>28.01</td>\n",
    "      <td>151</td>\n",
    "      <td>151</td>\n",
    "      <td>0</td>\n",
    "      <td>25.09</td>\n",
    "      <td>1755</td>\n",
    "      <td>1752</td>\n",
    "      <td>3</td>\n",
    "      <td>19.43</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(show_results(table, github=True).to_html(escape=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}