notebooks/evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# War of Words - Model Evaluation\n",
    "\n",
    "Use this notebook to train and evaluate the War of Words models:\n",
    "- With MEPs only (`WoW`)\n",
    "- With rapporteur advantage (`WoW(R)`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "from warofwords import WarOfWords, TrainedWarOfWords\n",
    "\n",
    "# Define all experiments.\n",
    "# (Sorry, I forgot to report these values in the paper...)\n",
    "experiments = [{\n",
    "    'data': '../data/processed/meponly-ep7-test.pkl',\n",
    "    'model': '../models/meponly-ep7.predict'\n",
    "}, {\n",
    "    'data': '../data/processed/rapadv-ep7-test.pkl',\n",
    "    'model': '../models/rapadv-ep7.predict'\n",
    "},{\n",
    "    'data': '../data/processed/meponly-ep8-test.pkl',\n",
    "    'model': '../models/meponly-ep8.predict'\n",
    "},{\n",
    "    'data': '../data/processed/rapadv-ep8-test.pkl',\n",
    "    'model': '../models/rapadv-ep8.predict'\n",
    "}]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, data):\n",
    "    \"\"\"Evaluate a model on a given test set.\"\"\"\n",
    "    \n",
    "    # Load data.\n",
    "    data = os.path.abspath(data)\n",
    "    print(f'  Test set: {data}')\n",
    "    features, featmats, labels = TrainedWarOfWords.load_data(data)\n",
    "    test = list(zip(featmats, labels))\n",
    "    \n",
    "    # Load trained model.\n",
    "    model = os.path.abspath(model)\n",
    "    print(f'  Model: {model}')\n",
    "    trained = TrainedWarOfWords.load(model)\n",
    "    \n",
    "    # Evaluate log loss.\n",
    "    print(f'  Log-loss: {trained.log_loss(test):.3f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment 1\n",
      "  Test set: data/processed/meponly-ep7-test.pkl\n",
      "  Model: models/meponly-ep7.predict\n",
      "  Log-loss: 0.714\n",
      "Experiment 2\n",
      "  Test set: data/processed/rapadv-ep7-test.pkl\n",
      "  Model: models/rapadv-ep7.predict\n",
      "  Log-loss: 0.690\n",
      "Experiment 3\n",
      "  Test set: data/processed/meponly-ep8-test.pkl\n",
      "  Model: models/meponly-ep8.predict\n",
      "  Log-loss: 0.748\n",
      "Experiment 4\n",
      "  Test set: data/processed/rapadv-ep8-test.pkl\n",
      "  Model: models/rapadv-ep8.predict\n",
      "  Log-loss: 0.726\n",
      "CPU times: user 6.78 s, sys: 2.33 s, total: 9.11 s\n",
      "Wall time: 9.97 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "for i, exp in enumerate(experiments):\n",
    "    print(f'Experiment {i+1}')\n",
    "    evaluate(exp['model'], exp['data'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}