cms_zee_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import GAN.cms_datasets as cms\n",
    "import GAN.plotting as plotting\n",
    "import GAN.preprocessing as preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C_NAMES': ['Phi', 'ScEta'],\n",
       " 'FEAT_TRANSFORM': u'minmax',\n",
       " 'LOAD_DATASETS': ['moriond_v9', 'abs(ScEta) < 1.5'],\n",
       " 'MCWEIGHT': u'weight',\n",
       " 'X_NAMES': ['EtaWidth', 'R9', 'SigmaIeIe', 'S4', 'PhiWidth', 'mass']}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import GAN.utils as utils\n",
    "\n",
    "# reload(utils)\n",
    "\n",
    "class Parameters(utils.Parameters):\n",
    "    \n",
    "    load_datasets=utils.param([\"moriond_v9\",\"abs(ScEta) < 1.5\"])\n",
    "\n",
    "    c_names = utils.param(['Phi','ScEta'])\n",
    "    x_names = utils.param(['EtaWidth','R9','SigmaIeIe','S4','PhiWidth','mass'])#,'Pt','mass'])\n",
    "\n",
    "#     # reweight = utils.param('rewei_zee_barrel.npy')\n",
    "#     reweight = utils.param(['rewei_zee_barrel.npy','rewei_zee_pt_barrel.npy'])#,'rewei_zee_pu_barrel.npy'])\n",
    "\n",
    "    mcweight = utils.param('weight')\n",
    "    \n",
    "    feat_transform = utils.param('minmax')\n",
    "#    feat_transform = utils.param('gaus')\n",
    "\n",
    "class MyApp(utils.MyApp):\n",
    "    classes = utils.List([Parameters])\n",
    "\n",
    "# Read all parameters above from command line. \n",
    "# Note: names are all converted to be all capital\n",
    "notebook_parameters = Parameters(MyApp()).get_params()\n",
    "\n",
    "# copy parameters to global scope\n",
    "globals().update(notebook_parameters)\n",
    "\n",
    "notebook_parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reload(cms)\n",
    "data,mc = cms.load_zee(*LOAD_DATASETS)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'index', u'run', u'rho', u'nvtx', u'mass', u'weight', u'SigMoM', u'Pt',\n",
       "       u'ScEta', u'Phi', u'R9', u'S4', u'SigmaIeIe', u'EtaWidth', u'PhiWidth',\n",
       "       u'CovarianceIphiIphi', u'SigmaRR', u'ScEnergy', u'CovarianceIetaIphi',\n",
       "       u'PhoIso03', u'ChIso03', u'ChIso03worst', u'ScPreshowerEnergy',\n",
       "       u'PhoIDMVA', u'SigEOverE', u'run_quantile'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'index', u'run', u'rho', u'nvtx', u'mass', u'weight', u'SigMoM', u'Pt',\n",
       "       u'ScEta', u'Phi', u'R9', u'S4', u'SigmaIeIe', u'EtaWidth', u'PhiWidth',\n",
       "       u'CovarianceIphiIphi', u'SigmaRR', u'ScEnergy', u'CovarianceIetaIphi',\n",
       "       u'PhoIso03', u'ChIso03', u'ChIso03worst', u'ScPreshowerEnergy',\n",
       "       u'PhoIDMVA', u'SigEOverE'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c_names = C_NAMES\n",
    "x_names = X_NAMES\n",
    "\n",
    "data_c = data[c_names]\n",
    "data_x = data[x_names]\n",
    "\n",
    "mc_c = mc[c_names]\n",
    "mc_x = mc[x_names]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index([u'EtaWidth', u'R9', u'SigmaIeIe', u'S4', u'PhiWidth', u'mass'], dtype='object'),\n",
       " (17520217, 6),\n",
       " Index([u'Phi', u'ScEta'], dtype='object'),\n",
       " (17520217, 2))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_x.columns, data_x.shape, data_c.columns, data_c.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index([u'EtaWidth', u'R9', u'SigmaIeIe', u'S4', u'PhiWidth', u'mass'], dtype='object'),\n",
       " Index([u'Phi', u'ScEta'], dtype='object'))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_x.columns, data_c.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Index([u'EtaWidth', u'R9', u'SigmaIeIe', u'S4', u'PhiWidth', u'mass'], dtype='object'),\n",
       " Index([u'Phi', u'ScEta'], dtype='object'))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc_x.columns, mc_c.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # reload(preprocessing)\n",
    "# if MCWEIGHT is None:\n",
    "#     mc_w = np.ones(mc_x.shape[0])\n",
    "# else:\n",
    "#     mc_w = mc[MCWEIGHT].values\n",
    "\n",
    "# if not REWEIGHT is None:\n",
    "#     for fil in REWEIGHT:\n",
    "#         info = np.load(fil)\n",
    "#         inputs = info[0]\n",
    "#         weights = info[1]\n",
    "#         bins = info[2:]\n",
    "#         # print(bins[1])\n",
    "#         print('weighting',inputs)\n",
    "#         mc_w *= preprocessing.reweight(mc,inputs,bins,weights,base=None)\n",
    "    \n",
    "# data_w = np.ones(data_x.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "q5 = data_x.quantile(0.05)\n",
    "q95 = data_x.quantile(0.95)\n",
    "data_min = data_x.min()\n",
    "data_max = data_x.max()\n",
    "\n",
    "iqr = data_x.quantile(0.75) - data_x.quantile(0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth      0.001511\n",
       "R9            0.096056\n",
       "SigmaIeIe     0.000000\n",
       "S4            0.115319\n",
       "PhiWidth      0.001289\n",
       "mass         70.000038\n",
       "dtype: float32"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_min"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth      0.006840\n",
       "R9            0.521665\n",
       "SigmaIeIe     0.007390\n",
       "S4            0.593159\n",
       "PhiWidth      0.009082\n",
       "mass         81.228424\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth      0.004387\n",
       "R9            0.323129\n",
       "SigmaIeIe     0.006670\n",
       "S4            0.463765\n",
       "PhiWidth     -0.025666\n",
       "mass         76.532669\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q5 - iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth      2.172245\n",
       "R9            2.143735\n",
       "SigmaIeIe    10.253435\n",
       "S4            3.692925\n",
       "PhiWidth      0.224259\n",
       "mass          2.391178\n",
       "dtype: float64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(q5 - data_min)/iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth        0.772673\n",
       "R9              9.882552\n",
       "SigmaIeIe       0.029891\n",
       "S4              0.988874\n",
       "PhiWidth        0.782234\n",
       "mass         1230.203613\n",
       "dtype: float32"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth       0.013756\n",
       "R9             0.966613\n",
       "SigmaIeIe      0.010354\n",
       "S4             0.916355\n",
       "PhiWidth       0.077592\n",
       "mass         109.754976\n",
       "dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth       0.016209\n",
       "R9             1.165149\n",
       "SigmaIeIe      0.011074\n",
       "S4             1.045748\n",
       "PhiWidth       0.112340\n",
       "mass         114.450731\n",
       "dtype: float64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q95 + iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth     309.341930\n",
       "R9            44.908390\n",
       "SigmaIeIe     27.105768\n",
       "S4             0.560458\n",
       "PhiWidth      20.279064\n",
       "mass         238.608836\n",
       "dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(data_max - q95)/iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thr = 1.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reject_low = data_x < q5 - thr*iqr\n",
    "reject_high = data_x > q95 + thr*iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "n_reject_low = reject_low.any(axis=1).sum()\n",
    "n_reject_high = reject_high.any(axis=1).sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_reject_low/data_x.shape[0], n_reject_high/data_x.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accept_data = ((data_x >= q5 - thr*iqr) & ( data_x <= q95 + thr*iqr )).all(axis=1)\n",
    "accept_mc   = ((mc_x   >= q5 - thr*iqr) & ( mc_x <= q95 + thr*iqr )).all(axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.91360175504675545"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accept_data.sum() / float(accept_data.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.94725635677823405"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accept_mc.sum() / float(accept_mc.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EtaWidth</th>\n",
       "      <th>R9</th>\n",
       "      <th>SigmaIeIe</th>\n",
       "      <th>S4</th>\n",
       "      <th>PhiWidth</th>\n",
       "      <th>mass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.600650e+07</td>\n",
       "      <td>1.600650e+07</td>\n",
       "      <td>1.600650e+07</td>\n",
       "      <td>1.600650e+07</td>\n",
       "      <td>1.600650e+07</td>\n",
       "      <td>1.600650e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>9.132168e-03</td>\n",
       "      <td>9.078916e-01</td>\n",
       "      <td>8.118966e-03</td>\n",
       "      <td>9.151140e-01</td>\n",
       "      <td>3.054186e-02</td>\n",
       "      <td>8.629321e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.834475e-03</td>\n",
       "      <td>1.536037e-01</td>\n",
       "      <td>9.883147e-04</td>\n",
       "      <td>1.327628e-01</td>\n",
       "      <td>2.184211e-02</td>\n",
       "      <td>6.117148e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.180516e-03</td>\n",
       "      <td>2.238724e-01</td>\n",
       "      <td>6.309268e-03</td>\n",
       "      <td>3.990729e-01</td>\n",
       "      <td>2.515367e-03</td>\n",
       "      <td>7.418481e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.909074e-03</td>\n",
       "      <td>7.631535e-01</td>\n",
       "      <td>8.528570e-03</td>\n",
       "      <td>7.777047e-01</td>\n",
       "      <td>1.229331e-02</td>\n",
       "      <td>8.759052e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>9.023006e-03</td>\n",
       "      <td>8.989664e-01</td>\n",
       "      <td>8.878669e-03</td>\n",
       "      <td>8.652633e-01</td>\n",
       "      <td>2.499229e-02</td>\n",
       "      <td>8.987257e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.024940e-02</td>\n",
       "      <td>9.479852e-01</td>\n",
       "      <td>9.210394e-03</td>\n",
       "      <td>8.969558e-01</td>\n",
       "      <td>4.568314e-02</td>\n",
       "      <td>9.190556e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.743610e-02</td>\n",
       "      <td>1.264299e+00</td>\n",
       "      <td>1.143475e-02</td>\n",
       "      <td>9.888741e-01</td>\n",
       "      <td>1.297113e-01</td>\n",
       "      <td>1.167986e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           EtaWidth            R9     SigmaIeIe            S4      PhiWidth  \\\n",
       "count  1.600650e+07  1.600650e+07  1.600650e+07  1.600650e+07  1.600650e+07   \n",
       "mean   9.132168e-03  9.078916e-01  8.118966e-03  9.151140e-01  3.054186e-02   \n",
       "std    1.834475e-03  1.536037e-01  9.883147e-04  1.327628e-01  2.184211e-02   \n",
       "min    3.180516e-03  2.238724e-01  6.309268e-03  3.990729e-01  2.515367e-03   \n",
       "25%    7.909074e-03  7.631535e-01  8.528570e-03  7.777047e-01  1.229331e-02   \n",
       "50%    9.023006e-03  8.989664e-01  8.878669e-03  8.652633e-01  2.499229e-02   \n",
       "75%    1.024940e-02  9.479852e-01  9.210394e-03  8.969558e-01  4.568314e-02   \n",
       "max    1.743610e-02  1.264299e+00  1.143475e-02  9.888741e-01  1.297113e-01   \n",
       "\n",
       "               mass  \n",
       "count  1.600650e+07  \n",
       "mean   8.629321e+01  \n",
       "std    6.117148e+00  \n",
       "min    7.418481e+01  \n",
       "25%    8.759052e+01  \n",
       "50%    8.987257e+01  \n",
       "75%    9.190556e+01  \n",
       "max    1.167986e+02  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_x[accept_data].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EtaWidth</th>\n",
       "      <th>R9</th>\n",
       "      <th>SigmaIeIe</th>\n",
       "      <th>S4</th>\n",
       "      <th>PhiWidth</th>\n",
       "      <th>mass</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>9.108999e+06</td>\n",
       "      <td>9.108999e+06</td>\n",
       "      <td>9.108999e+06</td>\n",
       "      <td>9.108999e+06</td>\n",
       "      <td>9.108999e+06</td>\n",
       "      <td>9.108999e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>8.740167e-03</td>\n",
       "      <td>8.732899e-01</td>\n",
       "      <td>8.335227e-03</td>\n",
       "      <td>8.667808e-01</td>\n",
       "      <td>2.976822e-02</td>\n",
       "      <td>8.386697e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.738866e-03</td>\n",
       "      <td>1.408023e-01</td>\n",
       "      <td>8.353423e-04</td>\n",
       "      <td>1.046282e-01</td>\n",
       "      <td>2.086873e-02</td>\n",
       "      <td>7.714291e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.494020e-03</td>\n",
       "      <td>2.239490e-01</td>\n",
       "      <td>6.309276e-03</td>\n",
       "      <td>3.990765e-01</td>\n",
       "      <td>1.274645e-06</td>\n",
       "      <td>7.418497e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>7.538203e-03</td>\n",
       "      <td>7.720036e-01</td>\n",
       "      <td>8.521758e-03</td>\n",
       "      <td>7.826572e-01</td>\n",
       "      <td>1.150721e-02</td>\n",
       "      <td>8.754033e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>8.561344e-03</td>\n",
       "      <td>9.062228e-01</td>\n",
       "      <td>8.873915e-03</td>\n",
       "      <td>8.691059e-01</td>\n",
       "      <td>2.228022e-02</td>\n",
       "      <td>8.985310e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>9.803479e-03</td>\n",
       "      <td>9.537584e-01</td>\n",
       "      <td>9.200254e-03</td>\n",
       "      <td>8.998990e-01</td>\n",
       "      <td>4.241024e-02</td>\n",
       "      <td>9.189629e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.743612e-02</td>\n",
       "      <td>1.263729e+00</td>\n",
       "      <td>1.143472e-02</td>\n",
       "      <td>9.940255e-01</td>\n",
       "      <td>1.297105e-01</td>\n",
       "      <td>1.167986e+02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           EtaWidth            R9     SigmaIeIe            S4      PhiWidth  \\\n",
       "count  9.108999e+06  9.108999e+06  9.108999e+06  9.108999e+06  9.108999e+06   \n",
       "mean   8.740167e-03  8.732899e-01  8.335227e-03  8.667808e-01  2.976822e-02   \n",
       "std    1.738866e-03  1.408023e-01  8.353423e-04  1.046282e-01  2.086873e-02   \n",
       "min    3.494020e-03  2.239490e-01  6.309276e-03  3.990765e-01  1.274645e-06   \n",
       "25%    7.538203e-03  7.720036e-01  8.521758e-03  7.826572e-01  1.150721e-02   \n",
       "50%    8.561344e-03  9.062228e-01  8.873915e-03  8.691059e-01  2.228022e-02   \n",
       "75%    9.803479e-03  9.537584e-01  9.200254e-03  8.998990e-01  4.241024e-02   \n",
       "max    1.743612e-02  1.263729e+00  1.143472e-02  9.940255e-01  1.297105e-01   \n",
       "\n",
       "               mass  \n",
       "count  9.108999e+06  \n",
       "mean   8.386697e+01  \n",
       "std    7.714291e+00  \n",
       "min    7.418497e+01  \n",
       "25%    8.754033e+01  \n",
       "50%    8.985310e+01  \n",
       "75%    9.189629e+01  \n",
       "max    1.167986e+02  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mc_x[accept_mc].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thr_up = q95+thr*iqr\n",
    "thr_down = q5-thr*iqr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth       0.017436\n",
       "R9             1.264417\n",
       "SigmaIeIe      0.011435\n",
       "S4             1.110445\n",
       "PhiWidth       0.129713\n",
       "mass         116.798608\n",
       "dtype: float64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thr_up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "EtaWidth      0.003160\n",
       "R9            0.223861\n",
       "SigmaIeIe     0.006309\n",
       "S4            0.399069\n",
       "PhiWidth     -0.043039\n",
       "mass         74.184792\n",
       "dtype: float64"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thr_down"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thr_up['mass'] = 110.\n",
    "thr_down['mass'] = 70.\n",
    "# thr_down['Pt'] = 25.\n",
    "# thr_up['Pt'] = 100."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "thr_up.to_hdf('cleaning_zee_m_barrel.hd5','thr_up',mode='w')\n",
    "thr_down.to_hdf('cleaning_zee_m_barrel.hd5','thr_down',mode='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'data_w' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-02d976d7f8c4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mmc_c\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmc_c\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maccept_mc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mdata_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_w\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maccept_data\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mmc_w\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmc_w\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0maccept_mc\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'data_w' is not defined"
     ]
    }
   ],
   "source": [
    "#data_x = data_x[accept_data]\n",
    "mc_x = mc_x[accept_mc]\n",
    "\n",
    "data_c = data_c[accept_data]\n",
    "mc_c = mc_c[accept_mc]\n",
    "\n",
    "data_w = data_w[accept_data]\n",
    "mc_w = mc_w[accept_mc]\n",
    "\n",
    "\n",
    "data_x,data_c,mc_x,mc_c,scaler_x,scaler_c = preprocessing.transform(data_x,data_c,mc_x,mc_c,FEAT_TRANSFORM)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unhashable type",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-49-0d8debaf1b2f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mix\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_names\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mplotting\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot_hists\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_x\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmc_x\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m#,range=[-3,3])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mxlabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_names\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/swshare/anaconda/lib/python2.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1995\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1996\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1997\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1998\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1999\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/swshare/anaconda/lib/python2.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_getitem_column\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2002\u001b[0m         \u001b[0;31m# get column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2003\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2004\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2006\u001b[0m         \u001b[0;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/swshare/anaconda/lib/python2.7/site-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   1346\u001b[0m         \u001b[0;34m\"\"\"Return the cached item, item represents a label indexer.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1347\u001b[0m         \u001b[0mcache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_item_cache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1348\u001b[0;31m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1349\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1350\u001b[0m             \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unhashable type"
     ]
    }
   ],
   "source": [
    "for ix in range(len(x_names)):\n",
    "    plotting.plot_hists(data_x[:,0,ix],mc_x[:,0,ix],bins=100)#,range=[-3,3])\n",
    "    plt.xlabel(x_names[ix])\n",
    "    plt.show()\n",
    "\n",
    "for ic in range(len(c_names)):\n",
    "    plotting.plot_hists(data_c[:,0,ic],mc_c[:,0,ic],bins=100)#,range=[-3,3])\n",
    "    plt.xlabel(c_names[ic])\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}