report.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# signac-benchmarks\n",
    "\n",
    "This notebook aggregates data from benchmark runs, which are currently implemented for signac and datreant.core.\n",
    "\n",
    "The reporting tool uses a signac Collection or a database as input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 1\n",
    "import sys\n",
    "from pprint import pprint\n",
    "from math import log, sqrt\n",
    "\n",
    "from signac import Collection, get_database\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "%aimport complexity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fetch the data\n",
    "\n",
    "Specify the data source and selection criteria.\n",
    "\n",
    " * `INPUT_SRC`: Where to fetch data from (either `db` or `file`).\n",
    " * `FN_COLLECTION`: Name of the local collection file.\n",
    " * `DB`: The name of the database that contains benchmark results.\n",
    " * `COLLECTION`: The name of the collection within the database that contains the benchmark results.\n",
    " * `QUERY`: A selection query for what data to include in the benchmark; should be considered across all measurements."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "INPUT_SRC = 'db'  # (db|file)\n",
    "FN_COLLECTION ='benchmark.txt'\n",
    "DB = 'testing'\n",
    "COLLECTION = 'signac_benchmarks'\n",
    "\n",
    "QUERY = {\n",
    "    'meta.N': {'$lte': 10000},   # increase or remove to include larger N\n",
    "    'meta.versions.python': {'$regex': '3.6.5'},\n",
    "    '$and': [\n",
    "        {'$or': [{'meta.tool': {'$ne': 'signac'}}, {'meta.versions.signac': '0.9.2'}]},\n",
    "        {'$or': [{'meta.tool': {'$ne': 'datreant'}}, {'meta.versions.datreant': '0.7.1'}]},\n",
    "    ],\n",
    "    'meta.cached': False,   # Specify, whether to include results with pre-caching.\n",
    "    'meta.profile': None,   # Make sure to exclude profiled runs.\n",
    "    'meta.fstype': 'nfs',   # nfs tmpfs hfs\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, load the data either from a database or a local collection file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of docs:  0\n"
     ]
    }
   ],
   "source": [
    "if INPUT_SRC == 'db':\n",
    "    db = get_database(DB)\n",
    "    docs = list(db[COLLECTION].find(QUERY))\n",
    "elif INPUT_SRC == 'file':\n",
    "    with Collection.open(FN_COLLECTION) as c:\n",
    "        docs = list(c.find(QUERY))\n",
    "else:\n",
    "    raise ValueError(INPUT_SRC)\n",
    "\n",
    "print(\"# of docs: \", len(docs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `complexity.COMPLEXITY` constant contains hard-coded (expected) complexities for the individual measured operations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'determine_len': 'N',\n",
      " 'iterate': 'N',\n",
      " 'iterate_single_pass': 'N',\n",
      " 'search_lean_filter': 'N',\n",
      " 'search_rich_filter': 'N',\n",
      " 'select_by_id': '1'}\n"
     ]
    }
   ],
   "source": [
    "pprint(complexity.COMPLEXITY)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specification of a few utility functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def tr(s):\n",
    "    cplx = complexity.COMPLEXITY.get(s)\n",
    "    t = {\n",
    "        'select_by_id': \"Select by ID\",\n",
    "        'determine_len': \"Determine N\",\n",
    "        'iterate': \"Iterate\",\n",
    "        'iterate_single_pass': \"Iterate (single pass)\",\n",
    "        'search_lean_filter': \"Search w/ lean filter\",\n",
    "        'search_rich_filter': \"Search w/ rich filter\",\n",
    "        'tool,N': \"Tool, $\\log_{10}(N)$\",\n",
    "        '3.6.3.final.0': \"3.6\",\n",
    "        'nfs': \"NFS\",\n",
    "    }.get(s, s)\n",
    "    if cplx is not None:\n",
    "        t += ' \\u2014 $\\mathcal{O}(' + '{})$'.format(cplx)\n",
    "    return t\n",
    "\n",
    "\n",
    "def tr_legend(legend):\n",
    "    title = legend.get_title()\n",
    "    title.set_text(tr(title.get_text()))\n",
    "    for text in legend.get_texts():\n",
    "        tool, N = text.get_text()[1:-1].split(',')\n",
    "        text.set_text(\"{}, {}\".format(tr(tool), round(log(int(N), 10))))\n",
    "\n",
    "        \n",
    "def get_versions(df, tool):\n",
    "    return set(df[df['tool'] == tool]['versions'].map(lambda v: v[tool]))\n",
    "\n",
    "\n",
    "def ensure_unique(iterable):\n",
    "    s = set(iterable)\n",
    "    if s:\n",
    "        if len(s) != 1:\n",
    "            raise ValueError(\"The following set is not unique: '{}'.\".format(s))\n",
    "        return s.pop()\n",
    "\n",
    "\n",
    "def get_meta(df):\n",
    "    signac_version = ensure_unique(get_versions(df, 'signac'))\n",
    "    if ensure_unique(df[df['tool'] == 'signac']['cached']):\n",
    "        signac_version += 'C'\n",
    "    return {\n",
    "        'python': ensure_unique(df['versions'].map(lambda v: v['python'])),\n",
    "        'signac': signac_version,\n",
    "        'datreant': ensure_unique(get_versions(df, 'datreant')),\n",
    "        'fstype': ensure_unique(df['fstype']),\n",
    "    }\n",
    "\n",
    "def fmt_meta(df):\n",
    "    return \"Python {} {}\".format(tr(meta['python']), tr(meta['fstype']))\n",
    "\n",
    "def fn_meta(df):\n",
    "    meta = get_meta(df)\n",
    "    keys = 'python', 'fstype', 'signac', 'datreant'\n",
    "    return '_'.join([k + '_{' + k + '}' for k in keys]).format(** meta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalization\n",
    "\n",
    "We average all runs within one session, select the shortest run, divide by the complexity order and finally convert the measured values to microseconds."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python_3.6.3.final.0_fstype_nfs_signac_0.9.2_datreant_0.7.1\n"
     ]
    }
   ],
   "source": [
    "def normalize(data, N):\n",
    "    for cat, x in data.items():\n",
    "        x_mean = min([(y/n) for n, y in x])\n",
    "        cplx = complexity.COMPLEXITY.get(cat)\n",
    "        if cplx is not None:\n",
    "            x_mean /= eval(cplx)\n",
    "        yield cat, x_mean * 1e6  # microseconds\n",
    "\n",
    "        \n",
    "df_meta = pd.DataFrame({doc['_id']: doc['meta'] for doc in docs}).T\n",
    "df_size = pd.DataFrame({doc['_id']: {'size': doc['size']['total']} for doc in docs}).T\n",
    "df_data = pd.DataFrame({doc['_id']: dict(normalize(doc['data'], doc['meta']['N'])) for doc in docs}).T\n",
    "\n",
    "df = pd.concat([df_meta, df_data, df_size], axis=1)\n",
    "\n",
    "# The identifier for this benchmark (also ensures consistency):\n",
    "print(fn_meta(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the benchmark values in microseconds and normalizd by complexity:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>tool</th>\n",
       "      <th colspan=\"3\" halign=\"left\">datreant</th>\n",
       "      <th colspan=\"3\" halign=\"left\">signac</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>N</th>\n",
       "      <th>100</th>\n",
       "      <th>1000</th>\n",
       "      <th>10000</th>\n",
       "      <th>100</th>\n",
       "      <th>1000</th>\n",
       "      <th>10000</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Determine N — $\\mathcal{O}(N)$</th>\n",
       "      <td>228.26</td>\n",
       "      <td>228.14</td>\n",
       "      <td>364.30</td>\n",
       "      <td>11.72</td>\n",
       "      <td>2.71</td>\n",
       "      <td>1.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Iterate — $\\mathcal{O}(N)$</th>\n",
       "      <td>20126.39</td>\n",
       "      <td>23473.49</td>\n",
       "      <td>26919.98</td>\n",
       "      <td>302.04</td>\n",
       "      <td>248.43</td>\n",
       "      <td>277.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Iterate (single pass) — $\\mathcal{O}(N)$</th>\n",
       "      <td>21618.97</td>\n",
       "      <td>22701.68</td>\n",
       "      <td>26953.80</td>\n",
       "      <td>1262.64</td>\n",
       "      <td>1284.08</td>\n",
       "      <td>1395.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Search w/ lean filter — $\\mathcal{O}(N)$</th>\n",
       "      <td>27685.41</td>\n",
       "      <td>28928.47</td>\n",
       "      <td>30203.04</td>\n",
       "      <td>122.83</td>\n",
       "      <td>103.42</td>\n",
       "      <td>122.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Search w/ rich filter — $\\mathcal{O}(N)$</th>\n",
       "      <td>160638.25</td>\n",
       "      <td>164399.46</td>\n",
       "      <td>NaN</td>\n",
       "      <td>147.27</td>\n",
       "      <td>131.89</td>\n",
       "      <td>144.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Select by ID — $\\mathcal{O}(1)$</th>\n",
       "      <td>19264.98</td>\n",
       "      <td>20248.56</td>\n",
       "      <td>19933.20</td>\n",
       "      <td>217.28</td>\n",
       "      <td>211.68</td>\n",
       "      <td>241.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>size</th>\n",
       "      <td>113600.00</td>\n",
       "      <td>1136000.00</td>\n",
       "      <td>11360000.00</td>\n",
       "      <td>113000.00</td>\n",
       "      <td>1130000.00</td>\n",
       "      <td>11300000.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "tool                                       datreant                           \\\n",
       "N                                             100         1000         10000   \n",
       "Determine N — $\\mathcal{O}(N)$               228.26      228.14       364.30   \n",
       "Iterate — $\\mathcal{O}(N)$                 20126.39    23473.49     26919.98   \n",
       "Iterate (single pass) — $\\mathcal{O}(N)$   21618.97    22701.68     26953.80   \n",
       "Search w/ lean filter — $\\mathcal{O}(N)$   27685.41    28928.47     30203.04   \n",
       "Search w/ rich filter — $\\mathcal{O}(N)$  160638.25   164399.46          NaN   \n",
       "Select by ID — $\\mathcal{O}(1)$            19264.98    20248.56     19933.20   \n",
       "size                                      113600.00  1136000.00  11360000.00   \n",
       "\n",
       "tool                                         signac                           \n",
       "N                                             100         1000         10000  \n",
       "Determine N — $\\mathcal{O}(N)$                11.72        2.71         1.41  \n",
       "Iterate — $\\mathcal{O}(N)$                   302.04      248.43       277.88  \n",
       "Iterate (single pass) — $\\mathcal{O}(N)$    1262.64     1284.08      1395.06  \n",
       "Search w/ lean filter — $\\mathcal{O}(N)$     122.83      103.42       122.28  \n",
       "Search w/ rich filter — $\\mathcal{O}(N)$     147.27      131.89       144.49  \n",
       "Select by ID — $\\mathcal{O}(1)$              217.28      211.68       241.84  \n",
       "size                                      113000.00  1130000.00  11300000.00  "
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.rename(columns=tr).groupby(['tool', 'N']).mean().round(2).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next, we prepare plotting the data by defining a tool and data space size ($N$) based color schemes.\n",
    "We use blue for signac and green for datreant, roughly corresponding to the color scheme on the respective websites."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "CMAP_TOOLS = {\n",
    "    'signac': plt.get_cmap('Blues'),\n",
    "    'datreant': plt.get_cmap('Greens'),\n",
    "}\n",
    "\n",
    "N_max = df['N'].max()\n",
    "\n",
    "def color(x, a_min=0.1, a_max=1.0):\n",
    "    tool = x['tool'][0]\n",
    "    N = x['N'][0]\n",
    "    y = log(N) / log(N_max)\n",
    "    y = a_min + y * (a_max - a_min)\n",
    "    return CMAP_TOOLS[tool](y)\n",
    "\n",
    "colors = df.groupby(['tool', 'N']).apply(color).tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following section plots the output data. The interactive offset values help to generate a \"publication-ready\" figure with a slighty customized (hard-coded) legend.\n",
    "The hard-coded legend only works for a specific subset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e7a58aa0cbf467ab1fa6fe0625bc8d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(FloatSlider(value=0.02, description='x_offset', max=1.0, step=0.01), FloatSlider(value=0…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.patches as patches\n",
    "from ipywidgets import interact\n",
    "\n",
    "offset = (0, 1.0, 0.01)\n",
    "\n",
    "@interact(x_offset=offset, y_offset=offset, x_offset_i=offset, y_offset_i=offset, y_lim_e=(1, 10, 1))\n",
    "def plot(x_offset=0.02, y_offset=0, x_offset_i=0, y_offset_i=0, y_lim_e=9, hardcoded_legend=True, save=False):\n",
    "    fig, ax = plt.subplots(figsize=(4, 3), dpi=150)\n",
    "    \n",
    "    meta = get_meta(df)\n",
    "    tools = df.sort_values('tool').groupby('tool').head(1)['tool'].tolist()\n",
    "    Ns = df.sort_values(['tool', 'N']).groupby(['tool', 'N']).head(1)['N'].tolist()\n",
    "    \n",
    "    if hardcoded_legend:\n",
    "        can_use_hcl = tools == ['datreant', 'signac'] and Ns == [100, 1000, 10000, 100, 1000, 10000]\n",
    "        if not can_use_hcl:\n",
    "            print(\"Can't use the hard-coded legend for this data set!\", file=sys.stderr)\n",
    "            hardcoded_legend = False\n",
    "    \n",
    "    tmp = df.rename(columns=tr).drop(columns=['size']).groupby(by=['tool', 'N']).mean()\n",
    "    tmp.T.plot(kind='barh', ax=ax, color=colors, log=True, legend=not hardcoded_legend, fontsize=8)\n",
    "\n",
    "    ax.set_xlim(1, 10**y_lim_e)\n",
    "    ax.set_xlabel('Execution Time / Complexity $(1|N)$ [\\u00B5s]')\n",
    "\n",
    "    if hardcoded_legend:\n",
    "        assert tools == ['datreant', 'signac'] and Ns == [100, 1000, 10000, 100, 1000, 10000]\n",
    "         \n",
    "        x = lambda x_: x_ + x_offset_i\n",
    "        y = lambda y_: y_ + y_offset_i\n",
    "\n",
    "        a = fig.add_axes([.6 + x_offset, .2 + y_offset, .25, .6], facecolor='w')\n",
    "        a.set_xticks([])\n",
    "        a.set_yticks([])\n",
    "        [spine.set_color('gray') for spine in a.spines.values()]\n",
    "#         plt.text(s='Tool', x=x(0.25), y = y(0.85), rotation=0, horizontalalignment='center')\n",
    "        plt.text(s='$\\log_{10}(N)$', x=x(0.67), y=y(0.85), rotation=0, horizontalalignment='center')\n",
    "    \n",
    "        plt.text(s='datreant', x=x(0.24), y = y(0.26), rotation=90, verticalalignment='center', horizontalalignment='center')\n",
    "        plt.text(s=str(meta['datreant']), x=x(0.37), y = y(0.26), size=6, rotation=90, verticalalignment='center', horizontalalignment='center')\n",
    "    \n",
    "        plt.text(s='signac', x=x(0.24), y = y(0.65), rotation=90, verticalalignment='center', horizontalalignment='center')\n",
    "        plt.text(s=str(meta['signac']), x=x(0.37), y=y(0.65), size=6, rotation=90, verticalalignment='center', horizontalalignment='center')\n",
    "        \n",
    "        logN = [2,3,4,2,3,4]\n",
    "        for i in range(6):\n",
    "            if(i>=3):\n",
    "                a.text(s='{}'.format(logN[i]), x=x(0.68), y=y(0.1*(i+2)+0.025), horizontalalignment='center', color = 'w')\n",
    "                a.add_patch(patches.Rectangle((x(0.48), y(0.1*(i+2))), 0.4, 0.1, color=colors[i]))\n",
    "            else:\n",
    "                a.text(s='{}'.format(logN[i]), x=x(0.68), y=y(0.1*(i+1)+0.025), horizontalalignment='center', color = 'w')\n",
    "                a.add_patch(patches.Rectangle((x(0.48), y(0.1*(i+1))), 0.4, 0.1, color=colors[i]))\n",
    "    else:\n",
    "        legend = ax.get_legend()\n",
    "        tr_legend(legend)\n",
    "        \n",
    "    fn = fn_meta(df) + '.pdf'\n",
    "    if save:\n",
    "        plt.savefig(fn, transparent=True, bbox_inches='tight')\n",
    "        print(\"Saved to '{}'.\".format(fn))\n",
    "    else:\n",
    "        print(\"Would save to '{}'.\".format(fn))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}