proc3-kstest.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing whether RTT KS value is properly computed from the DB pvals\n",
    "# Precond: all_subs is list of all subtest \n",
    "\n",
    "def pick_km_statistic(stats):\n",
    "    if len(stats) == 0:\n",
    "        return None\n",
    "    for st in ['kolm']:\n",
    "        for cur in stats:\n",
    "            name = cur.name.lower()\n",
    "            if name.startswith(st):\n",
    "                return cur\n",
    "    return None\n",
    "\n",
    "print('Len: %s' % len(all_subs))\n",
    "\n",
    "errvct = []\n",
    "stat_names_dct = collections.defaultdict(lambda: 0)\n",
    "lpvals = collections.defaultdict(lambda: 0)\n",
    "cursubs = []\n",
    "\n",
    "for st in all_subs:\n",
    "    for s in st.stats:\n",
    "        stat_names_dct[s.name] += 1\n",
    "    lpvals[len(st.pvals)] += 1\n",
    "    \n",
    "    if len(st.pvals) <= 1:\n",
    "        continue\n",
    "        \n",
    "    stat = pick_km_statistic(st.stats)\n",
    "    if stat is None:\n",
    "        continue  # Not a KS test\n",
    "        \n",
    "    cursubs.append(st)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#random.shuffle(cursubs)\n",
    "print(len(cursubs))\n",
    "\n",
    "errvct = []\n",
    "used_pval_len = []\n",
    "for st in cursubs[:1000]:       \n",
    "    stat = pick_km_statistic(st.stats)\n",
    "    ksres = stats.kstest(st.pvals, 'uniform')\n",
    "    errvct.append(ksres[1] - stat.value)\n",
    "    used_pval_len.append(len(st.pvals))\n",
    "    \n",
    "errvctab = [abs(x) for x in errvct]\n",
    "\n",
    "print('Num data: %s' % len(used_pval_len))\n",
    "print('Err min: %s, max: %s, Avg: %s, stddev: %s' % (min(errvctab), max(errvctab), np.average(errvctab), np.std(errvctab)))\n",
    "print(json.dumps(stat_names_dct, indent=2))\n",
    "\n",
    "print('Pvals min: %s, max: %s, avg: %s, stdev: %s' % (min(used_pval_len), max(used_pval_len), np.average(used_pval_len), np.std(used_pval_len)))\n",
    "fig, ax = plt.subplots()\n",
    "ax.set_xlabel('pval count')\n",
    "ax.set_ylabel('Counts')\n",
    "plt.hist(used_pval_len, log=True)\n",
    "plt.show()\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.hist(errvctab, bins=25, log=True)\n",
    "ax.set_xlabel('error |RTT_KS - KS|')\n",
    "ax.set_ylabel('Counts')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('0x08003d7e', 'ea46', '0141', 'orr.w r1, r6, r1, lsl #1')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.match(r'^\\s*(0x[0-9a-fA-F]+):\\s+([0-9a-fA-F]+)(?:\\s+([0-9a-fA-F]+))?\\s+(.+?)\\s*$', '0x08003d7e:  ea46 0141  orr.w r1, r6, r1, lsl #1').groups()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}