Build_analogues-all.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Created: 15.02.2016\n",
    "#Last modified: 18.02.2016\n",
    "#               29.02.2016 add Corr and Dis cols\n",
    "#               25.05.2016 updated input file and created truncated file for testing\n",
    "#               24.05.2016 tested reading in file produced in birdhouse analogs process\n",
    "#               05.09.2016 format test file for dc chart brush problem\n",
    "#               09.09.2016 added test file that spans two months to test date handling\n",
    "#                          change float format to 7 places in file save (float_format='%.7f')\n",
    "#               13.09.2016 replace hard-coded num_analogues value with var name\n",
    "#                          new test file with 11 analogues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_analogues = 20 #number of analogues searched for\n",
    "num_cols = 3 #dateAnlg, Dis, Corr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/homel/cnangini/PROJECTS/analogues'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date.an1</th>\n",
       "      <th>date.an2</th>\n",
       "      <th>date.an3</th>\n",
       "      <th>date.an4</th>\n",
       "      <th>date.an5</th>\n",
       "      <th>date.an6</th>\n",
       "      <th>date.an7</th>\n",
       "      <th>date.an8</th>\n",
       "      <th>date.an9</th>\n",
       "      <th>date.an10</th>\n",
       "      <th>...</th>\n",
       "      <th>cor11</th>\n",
       "      <th>cor12</th>\n",
       "      <th>cor13</th>\n",
       "      <th>cor14</th>\n",
       "      <th>cor15</th>\n",
       "      <th>cor16</th>\n",
       "      <th>cor17</th>\n",
       "      <th>cor18</th>\n",
       "      <th>cor19</th>\n",
       "      <th>cor20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 20110114</td>\n",
       "      <td> 19850130</td>\n",
       "      <td> 19980114</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19540119</td>\n",
       "      <td> 19870101</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.451300</td>\n",
       "      <td> 0.356393</td>\n",
       "      <td> 0.639460</td>\n",
       "      <td> 0.383338</td>\n",
       "      <td> 0.619769</td>\n",
       "      <td> 0.575454</td>\n",
       "      <td> 0.356389</td>\n",
       "      <td> 0.587782</td>\n",
       "      <td> 0.367948</td>\n",
       "      <td> 0.546730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 20110112</td>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19600121</td>\n",
       "      <td> 19741220</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19741219</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.508294</td>\n",
       "      <td> 0.500434</td>\n",
       "      <td> 0.667818</td>\n",
       "      <td> 0.454554</td>\n",
       "      <td> 0.560843</td>\n",
       "      <td> 0.438960</td>\n",
       "      <td> 0.326965</td>\n",
       "      <td> 0.455356</td>\n",
       "      <td> 0.478750</td>\n",
       "      <td> 0.595530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 19491206</td>\n",
       "      <td> 19741222</td>\n",
       "      <td> 19750110</td>\n",
       "      <td> 19641231</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19851222</td>\n",
       "      <td> 19741221</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19991211</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.645103</td>\n",
       "      <td> 0.554328</td>\n",
       "      <td> 0.729818</td>\n",
       "      <td> 0.448795</td>\n",
       "      <td> 0.410199</td>\n",
       "      <td> 0.518829</td>\n",
       "      <td> 0.639049</td>\n",
       "      <td> 0.647471</td>\n",
       "      <td> 0.452940</td>\n",
       "      <td> 0.492896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19891221</td>\n",
       "      <td> 19921216</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19491206</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19510114</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19550120</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.531300</td>\n",
       "      <td> 0.551238</td>\n",
       "      <td> 0.607714</td>\n",
       "      <td> 0.555354</td>\n",
       "      <td> 0.424680</td>\n",
       "      <td> 0.571842</td>\n",
       "      <td> 0.452101</td>\n",
       "      <td> 0.329466</td>\n",
       "      <td> 0.677823</td>\n",
       "      <td> 0.455432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 19971210</td>\n",
       "      <td> 19971209</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19971211</td>\n",
       "      <td> 19591230</td>\n",
       "      <td> 19591229</td>\n",
       "      <td> 19950129</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19550120</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.545885</td>\n",
       "      <td> 0.571774</td>\n",
       "      <td> 0.488988</td>\n",
       "      <td> 0.308756</td>\n",
       "      <td> 0.525776</td>\n",
       "      <td> 0.551664</td>\n",
       "      <td> 0.551487</td>\n",
       "      <td> 0.445159</td>\n",
       "      <td> 0.466376</td>\n",
       "      <td> 0.492252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480106</th>\n",
       "      <td> 20020122</td>\n",
       "      <td> 19710120</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19991211</td>\n",
       "      <td> 19591227</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19991210</td>\n",
       "      <td> 19950129</td>\n",
       "      <td> 19710126</td>\n",
       "      <td> 20040113</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.619992</td>\n",
       "      <td> 0.580125</td>\n",
       "      <td> 0.596970</td>\n",
       "      <td> 0.578637</td>\n",
       "      <td> 0.516381</td>\n",
       "      <td> 0.508655</td>\n",
       "      <td> 0.506890</td>\n",
       "      <td> 0.576995</td>\n",
       "      <td> 0.515145</td>\n",
       "      <td> 0.557619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480107</th>\n",
       "      <td> 19781214</td>\n",
       "      <td> 19591227</td>\n",
       "      <td> 19510112</td>\n",
       "      <td> 19811214</td>\n",
       "      <td> 19991211</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 20040114</td>\n",
       "      <td> 19991212</td>\n",
       "      <td> 19581213</td>\n",
       "      <td> 19571210</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.632526</td>\n",
       "      <td> 0.606254</td>\n",
       "      <td> 0.704889</td>\n",
       "      <td> 0.590619</td>\n",
       "      <td> 0.689622</td>\n",
       "      <td> 0.547238</td>\n",
       "      <td> 0.497128</td>\n",
       "      <td> 0.493634</td>\n",
       "      <td> 0.555382</td>\n",
       "      <td> 0.594444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480108</th>\n",
       "      <td> 20040114</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19781214</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19811213</td>\n",
       "      <td> 19510113</td>\n",
       "      <td> 19660102</td>\n",
       "      <td> 19740205</td>\n",
       "      <td> 19811214</td>\n",
       "      <td> 20030102</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.624003</td>\n",
       "      <td> 0.574805</td>\n",
       "      <td> 0.547084</td>\n",
       "      <td> 0.642704</td>\n",
       "      <td> 0.667571</td>\n",
       "      <td> 0.634750</td>\n",
       "      <td> 0.541569</td>\n",
       "      <td> 0.398236</td>\n",
       "      <td> 0.615544</td>\n",
       "      <td> 0.599792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480109</th>\n",
       "      <td> 20021230</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19861230</td>\n",
       "      <td> 19590114</td>\n",
       "      <td> 19660206</td>\n",
       "      <td> 19811213</td>\n",
       "      <td> 19660207</td>\n",
       "      <td> 19590113</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.624890</td>\n",
       "      <td> 0.436524</td>\n",
       "      <td> 0.451390</td>\n",
       "      <td> 0.531034</td>\n",
       "      <td> 0.440639</td>\n",
       "      <td> 0.490920</td>\n",
       "      <td> 0.497755</td>\n",
       "      <td> 0.519086</td>\n",
       "      <td> 0.424741</td>\n",
       "      <td> 0.440802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480110</th>\n",
       "      <td> 19550128</td>\n",
       "      <td> 19710116</td>\n",
       "      <td> 19800130</td>\n",
       "      <td> 19850126</td>\n",
       "      <td> 20030101</td>\n",
       "      <td> 20020122</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19590114</td>\n",
       "      <td> 19550129</td>\n",
       "      <td> 19861231</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.500901</td>\n",
       "      <td> 0.646659</td>\n",
       "      <td> 0.669257</td>\n",
       "      <td> 0.608757</td>\n",
       "      <td> 0.571818</td>\n",
       "      <td> 0.507996</td>\n",
       "      <td> 0.523361</td>\n",
       "      <td> 0.678171</td>\n",
       "      <td> 0.593135</td>\n",
       "      <td> 0.629520</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          date.an1  date.an2  date.an3  date.an4  date.an5  date.an6  \\\n",
       "date                                                                   \n",
       "19480101  19560126  19850131  20110114  19850130  19980114  19551224   \n",
       "19480102  19681221  20001211  19551224  20110112  19560126  19600121   \n",
       "19480103  19491206  19741222  19750110  19641231  19491207  19921215   \n",
       "19480104  20031217  19921215  19891221  19921216  19491207  19491206   \n",
       "19480105  19971210  19971209  19931217  19591228  19971211  19591230   \n",
       "19480106  20020122  19710120  19931217  19991211  19591227  20020121   \n",
       "19480107  19781214  19591227  19510112  19811214  19991211  19591228   \n",
       "19480108  20040114  19591228  19781214  19931217  19811213  19510113   \n",
       "19480109  20021230  20020121  19931217  19681220  19861230  19590114   \n",
       "19480110  19550128  19710116  19800130  19850126  20030101  20020122   \n",
       "\n",
       "          date.an7  date.an8  date.an9  date.an10    ...        cor11  \\\n",
       "date                                                 ...                \n",
       "19480101  19521221  19681220  19540119   19870101    ...     0.451300   \n",
       "19480102  19741220  19681220  19521221   19741219    ...     0.508294   \n",
       "19480103  19851222  19741221  19931217   19991211    ...     0.645103   \n",
       "19480104  20020121  19510114  19931217   19550120    ...     0.531300   \n",
       "19480105  19591229  19950129  20020121   19550120    ...     0.545885   \n",
       "19480106  19991210  19950129  19710126   20040113    ...     0.619992   \n",
       "19480107  20040114  19991212  19581213   19571210    ...     0.632526   \n",
       "19480108  19660102  19740205  19811214   20030102    ...     0.624003   \n",
       "19480109  19660206  19811213  19660207   19590113    ...     0.624890   \n",
       "19480110  20020121  19590114  19550129   19861231    ...     0.500901   \n",
       "\n",
       "             cor12     cor13     cor14     cor15     cor16     cor17  \\\n",
       "date                                                                   \n",
       "19480101  0.356393  0.639460  0.383338  0.619769  0.575454  0.356389   \n",
       "19480102  0.500434  0.667818  0.454554  0.560843  0.438960  0.326965   \n",
       "19480103  0.554328  0.729818  0.448795  0.410199  0.518829  0.639049   \n",
       "19480104  0.551238  0.607714  0.555354  0.424680  0.571842  0.452101   \n",
       "19480105  0.571774  0.488988  0.308756  0.525776  0.551664  0.551487   \n",
       "19480106  0.580125  0.596970  0.578637  0.516381  0.508655  0.506890   \n",
       "19480107  0.606254  0.704889  0.590619  0.689622  0.547238  0.497128   \n",
       "19480108  0.574805  0.547084  0.642704  0.667571  0.634750  0.541569   \n",
       "19480109  0.436524  0.451390  0.531034  0.440639  0.490920  0.497755   \n",
       "19480110  0.646659  0.669257  0.608757  0.571818  0.507996  0.523361   \n",
       "\n",
       "             cor18     cor19     cor20  \n",
       "date                                    \n",
       "19480101  0.587782  0.367948  0.546730  \n",
       "19480102  0.455356  0.478750  0.595530  \n",
       "19480103  0.647471  0.452940  0.492896  \n",
       "19480104  0.329466  0.677823  0.455432  \n",
       "19480105  0.445159  0.466376  0.492252  \n",
       "19480106  0.576995  0.515145  0.557619  \n",
       "19480107  0.493634  0.555382  0.594444  \n",
       "19480108  0.398236  0.615544  0.599792  \n",
       "19480109  0.519086  0.424741  0.440802  \n",
       "19480110  0.678171  0.593135  0.629520  \n",
       "\n",
       "[10 rows x 60 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#To read file produced in birdhouse\n",
    "dfS = pd.DataFrame()\n",
    "#analogs = 'http://birdhouse-lsce.extra.cea.fr:8090/wpsoutputs/flyingpigeon/output_txt-0797016c-378e-11e6-91dd-41d8cd554993.txt'\n",
    "#analogs = 'http://localhost:8090/wpsoutputs/flyingpigeon/analogs-c26ba398-711e-11e6-8d5a-fb3204a8dad1.txt'\n",
    "\n",
    "#Analysis period > 1 year\n",
    "#analogs = 'http://birdhouse-lsce.extra.cea.fr:8090/wpsoutputs/flyingpigeon/analogs-5ee81d98-6fc9-11e6-a296-4d3fe2c8a29e.txt'\n",
    "#analogs = 'http://birdhouse-lsce.extra.cea.fr:8090/wpsoutputs/flyingpigeon/analogs-73cd782c-74f4-11e6-bf5f-f73f2a3d7e35.txt'\n",
    "\n",
    "#bbox x: [-80,50], y: [22.5,70], 11 analogues\n",
    "#analogs = 'http://localhost:8090/wpsoutputs/flyingpigeon/analogs-12f189be-79a7-11e6-b7f7-e7ff4fd8b248.txt'\n",
    "\n",
    "#Analysis period spans two months (30.06.2013 to 02.07.2013)\n",
    "#analogs = 'http://localhost:8090/wpsoutputs/flyingpigeon/analogs-7aafc4ec-74ea-11e6-b77d-eb5606a7fe9f.txt'\n",
    "\n",
    "#one day only (03.01.2013)\n",
    "#analogs = 'http://localhost:8090/wpsoutputs/flyingpigeon/analogs-9841a808-7504-11e6-b77d-eb5606a7fe9f.txt'\n",
    "\n",
    "analogs = 'http://www-lscedods.cea.fr/DASE/ana_slp_surface_base_rms_NA_sim_1948-01-01_2016-09-23_base_1948-01-01_2016-09-23_-80.0_50.0_22.5_70.0_1_30_20.txt'\n",
    "\n",
    "dfS = pd.read_csv(analogs, delimiter=r\"\\s+\", index_col=0)\n",
    "dfS.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Create dataframe from analogues file\n",
    "# Latest file: /home/estimr1/yiou/estimr1/NCEP/slpano-NA.analog30rms.1d.all1948-2016.dat\n",
    "# 2015: /home/estimr1/yiou/estimr1/NCEP/slpano-NA.analog30rms.1d.all1948-2015.dat\n",
    "# Note that last day in 2015 file is 20151225 (and NOT 20151231) because that is the last sunday of Dec 2015.\n",
    "\n",
    "#dfS = pd.DataFrame()\n",
    "#dfS = pd.read_csv('/home/estimr1/yiou/estimr1/NCEP/slpano-NA.analog30rms.1d.all1948-2016.dat', delimiter=' ', index_col=0)\n",
    "#dfS.set_index('Date', inplace=True)\n",
    "\n",
    "#dfS.head(5)\n",
    "\n",
    "#dfS.index.name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date.an1</th>\n",
       "      <th>date.an2</th>\n",
       "      <th>date.an3</th>\n",
       "      <th>date.an4</th>\n",
       "      <th>date.an5</th>\n",
       "      <th>date.an6</th>\n",
       "      <th>date.an7</th>\n",
       "      <th>date.an8</th>\n",
       "      <th>date.an9</th>\n",
       "      <th>date.an10</th>\n",
       "      <th>...</th>\n",
       "      <th>cor11</th>\n",
       "      <th>cor12</th>\n",
       "      <th>cor13</th>\n",
       "      <th>cor14</th>\n",
       "      <th>cor15</th>\n",
       "      <th>cor16</th>\n",
       "      <th>cor17</th>\n",
       "      <th>cor18</th>\n",
       "      <th>cor19</th>\n",
       "      <th>cor20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20160917</th>\n",
       "      <td> 19550924</td>\n",
       "      <td> 19730828</td>\n",
       "      <td> 19790924</td>\n",
       "      <td> 19900901</td>\n",
       "      <td> 19900902</td>\n",
       "      <td> 20130825</td>\n",
       "      <td> 19900831</td>\n",
       "      <td> 19541004</td>\n",
       "      <td> 19670916</td>\n",
       "      <td> 19880823</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.494911</td>\n",
       "      <td> 0.530220</td>\n",
       "      <td> 0.523700</td>\n",
       "      <td> 0.550937</td>\n",
       "      <td> 0.587929</td>\n",
       "      <td> 0.578387</td>\n",
       "      <td> 0.476094</td>\n",
       "      <td> 0.553288</td>\n",
       "      <td> 0.686999</td>\n",
       "      <td> 0.637255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160918</th>\n",
       "      <td> 20050821</td>\n",
       "      <td> 19930927</td>\n",
       "      <td> 20050822</td>\n",
       "      <td> 19781001</td>\n",
       "      <td> 20130825</td>\n",
       "      <td> 19670912</td>\n",
       "      <td> 19840907</td>\n",
       "      <td> 19690907</td>\n",
       "      <td> 19730828</td>\n",
       "      <td> 20130911</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.546377</td>\n",
       "      <td> 0.613898</td>\n",
       "      <td> 0.554076</td>\n",
       "      <td> 0.519887</td>\n",
       "      <td> 0.467421</td>\n",
       "      <td> 0.558831</td>\n",
       "      <td> 0.578775</td>\n",
       "      <td> 0.528234</td>\n",
       "      <td> 0.560866</td>\n",
       "      <td> 0.569862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160919</th>\n",
       "      <td> 19930927</td>\n",
       "      <td> 19690907</td>\n",
       "      <td> 19670915</td>\n",
       "      <td> 19840907</td>\n",
       "      <td> 19590901</td>\n",
       "      <td> 19720928</td>\n",
       "      <td> 20050821</td>\n",
       "      <td> 19670912</td>\n",
       "      <td> 19690906</td>\n",
       "      <td> 19630906</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.536014</td>\n",
       "      <td> 0.645193</td>\n",
       "      <td> 0.533333</td>\n",
       "      <td> 0.542052</td>\n",
       "      <td> 0.556240</td>\n",
       "      <td> 0.540978</td>\n",
       "      <td> 0.601654</td>\n",
       "      <td> 0.551885</td>\n",
       "      <td> 0.506031</td>\n",
       "      <td> 0.541926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160920</th>\n",
       "      <td> 19590928</td>\n",
       "      <td> 19590904</td>\n",
       "      <td> 19970831</td>\n",
       "      <td> 19980829</td>\n",
       "      <td> 19990911</td>\n",
       "      <td> 19990914</td>\n",
       "      <td> 19670828</td>\n",
       "      <td> 19670821</td>\n",
       "      <td> 19810916</td>\n",
       "      <td> 19930927</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.516680</td>\n",
       "      <td> 0.449324</td>\n",
       "      <td> 0.584607</td>\n",
       "      <td> 0.629824</td>\n",
       "      <td> 0.501016</td>\n",
       "      <td> 0.350878</td>\n",
       "      <td> 0.452059</td>\n",
       "      <td> 0.543318</td>\n",
       "      <td> 0.532147</td>\n",
       "      <td> 0.530831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19970831</td>\n",
       "      <td> 19810912</td>\n",
       "      <td> 19970901</td>\n",
       "      <td> 19680926</td>\n",
       "      <td> 19670828</td>\n",
       "      <td> 20010823</td>\n",
       "      <td> 19990912</td>\n",
       "      <td> 20020822</td>\n",
       "      <td> 20130904</td>\n",
       "      <td> 19790906</td>\n",
       "      <td>...</td>\n",
       "      <td> 0.469278</td>\n",
       "      <td> 0.547674</td>\n",
       "      <td> 0.493447</td>\n",
       "      <td> 0.568598</td>\n",
       "      <td> 0.621988</td>\n",
       "      <td> 0.470806</td>\n",
       "      <td> 0.484619</td>\n",
       "      <td> 0.415522</td>\n",
       "      <td> 0.610709</td>\n",
       "      <td> 0.456672</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          date.an1  date.an2  date.an3  date.an4  date.an5  date.an6  \\\n",
       "date                                                                   \n",
       "20160917  19550924  19730828  19790924  19900901  19900902  20130825   \n",
       "20160918  20050821  19930927  20050822  19781001  20130825  19670912   \n",
       "20160919  19930927  19690907  19670915  19840907  19590901  19720928   \n",
       "20160920  19590928  19590904  19970831  19980829  19990911  19990914   \n",
       "20160921  19970831  19810912  19970901  19680926  19670828  20010823   \n",
       "\n",
       "          date.an7  date.an8  date.an9  date.an10    ...        cor11  \\\n",
       "date                                                 ...                \n",
       "20160917  19900831  19541004  19670916   19880823    ...     0.494911   \n",
       "20160918  19840907  19690907  19730828   20130911    ...     0.546377   \n",
       "20160919  20050821  19670912  19690906   19630906    ...     0.536014   \n",
       "20160920  19670828  19670821  19810916   19930927    ...     0.516680   \n",
       "20160921  19990912  20020822  20130904   19790906    ...     0.469278   \n",
       "\n",
       "             cor12     cor13     cor14     cor15     cor16     cor17  \\\n",
       "date                                                                   \n",
       "20160917  0.530220  0.523700  0.550937  0.587929  0.578387  0.476094   \n",
       "20160918  0.613898  0.554076  0.519887  0.467421  0.558831  0.578775   \n",
       "20160919  0.645193  0.533333  0.542052  0.556240  0.540978  0.601654   \n",
       "20160920  0.449324  0.584607  0.629824  0.501016  0.350878  0.452059   \n",
       "20160921  0.547674  0.493447  0.568598  0.621988  0.470806  0.484619   \n",
       "\n",
       "             cor18     cor19     cor20  \n",
       "date                                    \n",
       "20160917  0.553288  0.686999  0.637255  \n",
       "20160918  0.528234  0.560866  0.569862  \n",
       "20160919  0.551885  0.506031  0.541926  \n",
       "20160920  0.543318  0.532147  0.530831  \n",
       "20160921  0.415522  0.610709  0.456672  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Int64Index([19480101, 19480102, 19480103, 19480104, 19480105, 19480106, 19480107, 19480108, 19480109, 19480110, 19480111, 19480112, 19480113, 19480114, 19480115, 19480116, 19480117, 19480118, 19480119, 19480120, 19480121, 19480122, 19480123, 19480124, 19480125, 19480126, 19480127, 19480128, 19480129, 19480130, 19480131, 19480201, 19480202, 19480203, 19480204, 19480205, 19480206, 19480207, 19480208, 19480209, 19480210, 19480211, 19480212, 19480213, 19480214, 19480215, 19480216, 19480217, 19480218, 19480219, 19480220, 19480221, 19480222, 19480223, 19480224, 19480225, 19480226, 19480227, 19480228, 19480229, 19480301, 19480302, 19480303, 19480304, 19480305, 19480306, 19480307, 19480308, 19480309, 19480310, 19480311, 19480312, 19480313, 19480314, 19480315, 19480316, 19480317, 19480318, 19480319, 19480320, 19480321, 19480322, 19480323, 19480324, 19480325, 19480326, 19480327, 19480328, 19480329, 19480330, 19480331, 19480401, 19480402, 19480403, 19480404, 19480405, 19480406, 19480407, 19480408, 19480409, ...], dtype='int64')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25102"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dfS.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25102, 60)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(dfS.shape[1])/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Chop off 2016 rows NO!!! there will still be 2016 dates found as analogues, which messes up js script\n",
    "#dfS = dfS[dfS.index <= 20151231]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "date\n",
       "19480101    19480101\n",
       "19480102    19480102\n",
       "19480103    19480103\n",
       "19480104    19480104\n",
       "19480105    19480105\n",
       "19480106    19480106\n",
       "19480107    19480107\n",
       "19480108    19480108\n",
       "19480109    19480109\n",
       "19480110    19480110\n",
       "Name: dateRef, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#SAVE DATE INDEX\n",
    "#DF['id2'] = DF.index\n",
    "dfS['dateRef'] = dfS.index\n",
    "dfS['dateRef'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date.an1</th>\n",
       "      <th>date.an2</th>\n",
       "      <th>date.an3</th>\n",
       "      <th>date.an4</th>\n",
       "      <th>date.an5</th>\n",
       "      <th>date.an6</th>\n",
       "      <th>date.an7</th>\n",
       "      <th>date.an8</th>\n",
       "      <th>date.an9</th>\n",
       "      <th>date.an10</th>\n",
       "      <th>date.an11</th>\n",
       "      <th>date.an12</th>\n",
       "      <th>date.an13</th>\n",
       "      <th>date.an14</th>\n",
       "      <th>date.an15</th>\n",
       "      <th>date.an16</th>\n",
       "      <th>date.an17</th>\n",
       "      <th>date.an18</th>\n",
       "      <th>date.an19</th>\n",
       "      <th>date.an20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 20110114</td>\n",
       "      <td> 19850130</td>\n",
       "      <td> 19980114</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19540119</td>\n",
       "      <td> 19870101</td>\n",
       "      <td> 19871224</td>\n",
       "      <td> 19610109</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 20010108</td>\n",
       "      <td> 20110112</td>\n",
       "      <td> 19560125</td>\n",
       "      <td> 19651223</td>\n",
       "      <td> 19651213</td>\n",
       "      <td> 20031209</td>\n",
       "      <td> 20110113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 20110112</td>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19600121</td>\n",
       "      <td> 19741220</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19741219</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 19670126</td>\n",
       "      <td> 19681222</td>\n",
       "      <td> 20110114</td>\n",
       "      <td> 19980108</td>\n",
       "      <td> 19670127</td>\n",
       "      <td> 19971223</td>\n",
       "      <td> 19980114</td>\n",
       "      <td> 19851223</td>\n",
       "      <td> 19670128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 19491206</td>\n",
       "      <td> 19741222</td>\n",
       "      <td> 19750110</td>\n",
       "      <td> 19641231</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19851222</td>\n",
       "      <td> 19741221</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19991211</td>\n",
       "      <td> 19991212</td>\n",
       "      <td> 19750109</td>\n",
       "      <td> 19950124</td>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921216</td>\n",
       "      <td> 20020120</td>\n",
       "      <td> 19991213</td>\n",
       "      <td> 19921214</td>\n",
       "      <td> 19640202</td>\n",
       "      <td> 19681220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19891221</td>\n",
       "      <td> 19921216</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19491206</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19510114</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19550120</td>\n",
       "      <td> 19741224</td>\n",
       "      <td> 19851222</td>\n",
       "      <td> 19641231</td>\n",
       "      <td> 19931218</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19741223</td>\n",
       "      <td> 19651219</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 19991213</td>\n",
       "      <td> 19921206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 19971210</td>\n",
       "      <td> 19971209</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19971211</td>\n",
       "      <td> 19591230</td>\n",
       "      <td> 19591229</td>\n",
       "      <td> 19950129</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19550120</td>\n",
       "      <td> 19861228</td>\n",
       "      <td> 19931218</td>\n",
       "      <td> 19741224</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 20020122</td>\n",
       "      <td> 19861230</td>\n",
       "      <td> 19680115</td>\n",
       "      <td> 19510114</td>\n",
       "      <td> 20010104</td>\n",
       "      <td> 20020120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          date.an1  date.an2  date.an3  date.an4  date.an5  date.an6  \\\n",
       "date                                                                   \n",
       "19480101  19560126  19850131  20110114  19850130  19980114  19551224   \n",
       "19480102  19681221  20001211  19551224  20110112  19560126  19600121   \n",
       "19480103  19491206  19741222  19750110  19641231  19491207  19921215   \n",
       "19480104  20031217  19921215  19891221  19921216  19491207  19491206   \n",
       "19480105  19971210  19971209  19931217  19591228  19971211  19591230   \n",
       "\n",
       "          date.an7  date.an8  date.an9  date.an10  date.an11  date.an12  \\\n",
       "date                                                                      \n",
       "19480101  19521221  19681220  19540119   19870101   19871224   19610109   \n",
       "19480102  19741220  19681220  19521221   19741219   19850131   19670126   \n",
       "19480103  19851222  19741221  19931217   19991211   19991212   19750109   \n",
       "19480104  20020121  19510114  19931217   19550120   19741224   19851222   \n",
       "19480105  19591229  19950129  20020121   19550120   19861228   19931218   \n",
       "\n",
       "          date.an13  date.an14  date.an15  date.an16  date.an17  date.an18  \\\n",
       "date                                                                         \n",
       "19480101   20001211   20010108   20110112   19560125   19651223   19651213   \n",
       "19480102   19681222   20110114   19980108   19670127   19971223   19980114   \n",
       "19480103   19950124   20031217   19921216   20020120   19991213   19921214   \n",
       "19480104   19641231   19931218   19681220   19741223   19651219   19551224   \n",
       "19480105   19741224   19551224   20020122   19861230   19680115   19510114   \n",
       "\n",
       "          date.an19  date.an20  \n",
       "date                            \n",
       "19480101   20031209   20110113  \n",
       "19480102   19851223   19670128  \n",
       "19480103   19640202   19681220  \n",
       "19480104   19991213   19921206  \n",
       "19480105   20010104   20020120  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.iloc[0:5, 0:num_analogues]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date.an1</th>\n",
       "      <th>date.an2</th>\n",
       "      <th>date.an3</th>\n",
       "      <th>date.an4</th>\n",
       "      <th>date.an5</th>\n",
       "      <th>date.an6</th>\n",
       "      <th>date.an7</th>\n",
       "      <th>date.an8</th>\n",
       "      <th>date.an9</th>\n",
       "      <th>date.an10</th>\n",
       "      <th>...</th>\n",
       "      <th>date.an13</th>\n",
       "      <th>date.an14</th>\n",
       "      <th>date.an15</th>\n",
       "      <th>date.an16</th>\n",
       "      <th>date.an17</th>\n",
       "      <th>date.an18</th>\n",
       "      <th>date.an19</th>\n",
       "      <th>date.an20</th>\n",
       "      <th>dis1</th>\n",
       "      <th>dis2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 20110114</td>\n",
       "      <td> 19850130</td>\n",
       "      <td> 19980114</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19540119</td>\n",
       "      <td> 19870101</td>\n",
       "      <td>...</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 20010108</td>\n",
       "      <td> 20110112</td>\n",
       "      <td> 19560125</td>\n",
       "      <td> 19651223</td>\n",
       "      <td> 19651213</td>\n",
       "      <td> 20031209</td>\n",
       "      <td> 20110113</td>\n",
       "      <td> 515.034</td>\n",
       "      <td> 596.280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 20110112</td>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19600121</td>\n",
       "      <td> 19741220</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19521221</td>\n",
       "      <td> 19741219</td>\n",
       "      <td>...</td>\n",
       "      <td> 19681222</td>\n",
       "      <td> 20110114</td>\n",
       "      <td> 19980108</td>\n",
       "      <td> 19670127</td>\n",
       "      <td> 19971223</td>\n",
       "      <td> 19980114</td>\n",
       "      <td> 19851223</td>\n",
       "      <td> 19670128</td>\n",
       "      <td> 692.016</td>\n",
       "      <td> 699.794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 19491206</td>\n",
       "      <td> 19741222</td>\n",
       "      <td> 19750110</td>\n",
       "      <td> 19641231</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19851222</td>\n",
       "      <td> 19741221</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19991211</td>\n",
       "      <td>...</td>\n",
       "      <td> 19950124</td>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921216</td>\n",
       "      <td> 20020120</td>\n",
       "      <td> 19991213</td>\n",
       "      <td> 19921214</td>\n",
       "      <td> 19640202</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 668.841</td>\n",
       "      <td> 771.007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19891221</td>\n",
       "      <td> 19921216</td>\n",
       "      <td> 19491207</td>\n",
       "      <td> 19491206</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19510114</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19550120</td>\n",
       "      <td>...</td>\n",
       "      <td> 19641231</td>\n",
       "      <td> 19931218</td>\n",
       "      <td> 19681220</td>\n",
       "      <td> 19741223</td>\n",
       "      <td> 19651219</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 19991213</td>\n",
       "      <td> 19921206</td>\n",
       "      <td> 740.066</td>\n",
       "      <td> 799.577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 19971210</td>\n",
       "      <td> 19971209</td>\n",
       "      <td> 19931217</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19971211</td>\n",
       "      <td> 19591230</td>\n",
       "      <td> 19591229</td>\n",
       "      <td> 19950129</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19550120</td>\n",
       "      <td>...</td>\n",
       "      <td> 19741224</td>\n",
       "      <td> 19551224</td>\n",
       "      <td> 20020122</td>\n",
       "      <td> 19861230</td>\n",
       "      <td> 19680115</td>\n",
       "      <td> 19510114</td>\n",
       "      <td> 20010104</td>\n",
       "      <td> 20020120</td>\n",
       "      <td> 767.951</td>\n",
       "      <td> 788.899</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          date.an1  date.an2  date.an3  date.an4  date.an5  date.an6  \\\n",
       "date                                                                   \n",
       "19480101  19560126  19850131  20110114  19850130  19980114  19551224   \n",
       "19480102  19681221  20001211  19551224  20110112  19560126  19600121   \n",
       "19480103  19491206  19741222  19750110  19641231  19491207  19921215   \n",
       "19480104  20031217  19921215  19891221  19921216  19491207  19491206   \n",
       "19480105  19971210  19971209  19931217  19591228  19971211  19591230   \n",
       "\n",
       "          date.an7  date.an8  date.an9  date.an10   ...     date.an13  \\\n",
       "date                                                ...                 \n",
       "19480101  19521221  19681220  19540119   19870101   ...      20001211   \n",
       "19480102  19741220  19681220  19521221   19741219   ...      19681222   \n",
       "19480103  19851222  19741221  19931217   19991211   ...      19950124   \n",
       "19480104  20020121  19510114  19931217   19550120   ...      19641231   \n",
       "19480105  19591229  19950129  20020121   19550120   ...      19741224   \n",
       "\n",
       "          date.an14  date.an15  date.an16  date.an17  date.an18  date.an19  \\\n",
       "date                                                                         \n",
       "19480101   20010108   20110112   19560125   19651223   19651213   20031209   \n",
       "19480102   20110114   19980108   19670127   19971223   19980114   19851223   \n",
       "19480103   20031217   19921216   20020120   19991213   19921214   19640202   \n",
       "19480104   19931218   19681220   19741223   19651219   19551224   19991213   \n",
       "19480105   19551224   20020122   19861230   19680115   19510114   20010104   \n",
       "\n",
       "          date.an20     dis1     dis2  \n",
       "date                                   \n",
       "19480101   20110113  515.034  596.280  \n",
       "19480102   19670128  692.016  699.794  \n",
       "19480103   19681220  668.841  771.007  \n",
       "19480104   19921206  740.066  799.577  \n",
       "19480105   20020120  767.951  788.899  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = num_analogues + 2\n",
    "dfS.iloc[0:5, 0:tmp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dis1</th>\n",
       "      <th>dis2</th>\n",
       "      <th>dis3</th>\n",
       "      <th>dis4</th>\n",
       "      <th>dis5</th>\n",
       "      <th>dis6</th>\n",
       "      <th>dis7</th>\n",
       "      <th>dis8</th>\n",
       "      <th>dis9</th>\n",
       "      <th>dis10</th>\n",
       "      <th>dis11</th>\n",
       "      <th>dis12</th>\n",
       "      <th>dis13</th>\n",
       "      <th>dis14</th>\n",
       "      <th>dis15</th>\n",
       "      <th>dis16</th>\n",
       "      <th>dis17</th>\n",
       "      <th>dis18</th>\n",
       "      <th>dis19</th>\n",
       "      <th>dis20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 515.034</td>\n",
       "      <td> 596.280</td>\n",
       "      <td> 617.760</td>\n",
       "      <td> 634.687</td>\n",
       "      <td> 637.823</td>\n",
       "      <td> 638.138</td>\n",
       "      <td> 645.760</td>\n",
       "      <td> 647.538</td>\n",
       "      <td> 648.427</td>\n",
       "      <td> 650.624</td>\n",
       "      <td> 653.039</td>\n",
       "      <td> 658.585</td>\n",
       "      <td> 666.854</td>\n",
       "      <td> 667.827</td>\n",
       "      <td> 668.390</td>\n",
       "      <td> 674.883</td>\n",
       "      <td> 680.150</td>\n",
       "      <td> 683.547</td>\n",
       "      <td> 686.121</td>\n",
       "      <td> 686.633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 692.016</td>\n",
       "      <td> 699.794</td>\n",
       "      <td> 712.922</td>\n",
       "      <td> 742.944</td>\n",
       "      <td> 755.533</td>\n",
       "      <td> 760.423</td>\n",
       "      <td> 767.120</td>\n",
       "      <td> 768.406</td>\n",
       "      <td> 787.050</td>\n",
       "      <td> 797.082</td>\n",
       "      <td> 797.162</td>\n",
       "      <td> 799.264</td>\n",
       "      <td> 813.808</td>\n",
       "      <td> 815.246</td>\n",
       "      <td> 816.868</td>\n",
       "      <td> 819.609</td>\n",
       "      <td> 819.895</td>\n",
       "      <td> 821.457</td>\n",
       "      <td> 822.148</td>\n",
       "      <td> 822.886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 668.841</td>\n",
       "      <td> 771.007</td>\n",
       "      <td> 788.757</td>\n",
       "      <td> 793.463</td>\n",
       "      <td> 793.981</td>\n",
       "      <td> 794.718</td>\n",
       "      <td> 797.097</td>\n",
       "      <td> 809.283</td>\n",
       "      <td> 811.915</td>\n",
       "      <td> 813.207</td>\n",
       "      <td> 814.863</td>\n",
       "      <td> 823.584</td>\n",
       "      <td> 825.919</td>\n",
       "      <td> 827.578</td>\n",
       "      <td> 827.972</td>\n",
       "      <td> 829.743</td>\n",
       "      <td> 830.399</td>\n",
       "      <td> 833.843</td>\n",
       "      <td> 837.687</td>\n",
       "      <td> 845.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 740.066</td>\n",
       "      <td> 799.577</td>\n",
       "      <td> 806.550</td>\n",
       "      <td> 808.457</td>\n",
       "      <td> 811.225</td>\n",
       "      <td> 816.784</td>\n",
       "      <td> 818.396</td>\n",
       "      <td> 821.668</td>\n",
       "      <td> 822.509</td>\n",
       "      <td> 825.544</td>\n",
       "      <td> 826.043</td>\n",
       "      <td> 828.015</td>\n",
       "      <td> 831.074</td>\n",
       "      <td> 836.530</td>\n",
       "      <td> 837.065</td>\n",
       "      <td> 837.426</td>\n",
       "      <td> 851.314</td>\n",
       "      <td> 854.682</td>\n",
       "      <td> 854.854</td>\n",
       "      <td> 856.017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 767.951</td>\n",
       "      <td> 788.899</td>\n",
       "      <td> 812.595</td>\n",
       "      <td> 814.832</td>\n",
       "      <td> 816.054</td>\n",
       "      <td> 817.158</td>\n",
       "      <td> 824.976</td>\n",
       "      <td> 836.605</td>\n",
       "      <td> 837.994</td>\n",
       "      <td> 838.440</td>\n",
       "      <td> 852.658</td>\n",
       "      <td> 854.182</td>\n",
       "      <td> 856.888</td>\n",
       "      <td> 863.888</td>\n",
       "      <td> 864.995</td>\n",
       "      <td> 868.510</td>\n",
       "      <td> 868.862</td>\n",
       "      <td> 869.480</td>\n",
       "      <td> 869.800</td>\n",
       "      <td> 871.417</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             dis1     dis2     dis3     dis4     dis5     dis6     dis7  \\\n",
       "date                                                                      \n",
       "19480101  515.034  596.280  617.760  634.687  637.823  638.138  645.760   \n",
       "19480102  692.016  699.794  712.922  742.944  755.533  760.423  767.120   \n",
       "19480103  668.841  771.007  788.757  793.463  793.981  794.718  797.097   \n",
       "19480104  740.066  799.577  806.550  808.457  811.225  816.784  818.396   \n",
       "19480105  767.951  788.899  812.595  814.832  816.054  817.158  824.976   \n",
       "\n",
       "             dis8     dis9    dis10    dis11    dis12    dis13    dis14  \\\n",
       "date                                                                      \n",
       "19480101  647.538  648.427  650.624  653.039  658.585  666.854  667.827   \n",
       "19480102  768.406  787.050  797.082  797.162  799.264  813.808  815.246   \n",
       "19480103  809.283  811.915  813.207  814.863  823.584  825.919  827.578   \n",
       "19480104  821.668  822.509  825.544  826.043  828.015  831.074  836.530   \n",
       "19480105  836.605  837.994  838.440  852.658  854.182  856.888  863.888   \n",
       "\n",
       "            dis15    dis16    dis17    dis18    dis19    dis20  \n",
       "date                                                            \n",
       "19480101  668.390  674.883  680.150  683.547  686.121  686.633  \n",
       "19480102  816.868  819.609  819.895  821.457  822.148  822.886  \n",
       "19480103  827.972  829.743  830.399  833.843  837.687  845.400  \n",
       "19480104  837.065  837.426  851.314  854.682  854.854  856.017  \n",
       "19480105  864.995  868.510  868.862  869.480  869.800  871.417  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.iloc[0:5, num_analogues:2*num_analogues]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cor1</th>\n",
       "      <th>cor2</th>\n",
       "      <th>cor3</th>\n",
       "      <th>cor4</th>\n",
       "      <th>cor5</th>\n",
       "      <th>cor6</th>\n",
       "      <th>cor7</th>\n",
       "      <th>cor8</th>\n",
       "      <th>cor9</th>\n",
       "      <th>cor10</th>\n",
       "      <th>cor11</th>\n",
       "      <th>cor12</th>\n",
       "      <th>cor13</th>\n",
       "      <th>cor14</th>\n",
       "      <th>cor15</th>\n",
       "      <th>cor16</th>\n",
       "      <th>cor17</th>\n",
       "      <th>cor18</th>\n",
       "      <th>cor19</th>\n",
       "      <th>cor20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 0.726977</td>\n",
       "      <td> 0.652806</td>\n",
       "      <td> 0.623012</td>\n",
       "      <td> 0.506846</td>\n",
       "      <td> 0.523203</td>\n",
       "      <td> 0.457494</td>\n",
       "      <td> 0.426564</td>\n",
       "      <td> 0.649643</td>\n",
       "      <td> 0.608331</td>\n",
       "      <td> 0.493753</td>\n",
       "      <td> 0.451300</td>\n",
       "      <td> 0.356393</td>\n",
       "      <td> 0.639460</td>\n",
       "      <td> 0.383338</td>\n",
       "      <td> 0.619769</td>\n",
       "      <td> 0.575454</td>\n",
       "      <td> 0.356389</td>\n",
       "      <td> 0.587782</td>\n",
       "      <td> 0.367948</td>\n",
       "      <td> 0.546730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 0.760688</td>\n",
       "      <td> 0.604350</td>\n",
       "      <td> 0.564686</td>\n",
       "      <td> 0.636293</td>\n",
       "      <td> 0.549962</td>\n",
       "      <td> 0.649019</td>\n",
       "      <td> 0.635242</td>\n",
       "      <td> 0.609948</td>\n",
       "      <td> 0.445098</td>\n",
       "      <td> 0.539979</td>\n",
       "      <td> 0.508294</td>\n",
       "      <td> 0.500434</td>\n",
       "      <td> 0.667818</td>\n",
       "      <td> 0.454554</td>\n",
       "      <td> 0.560843</td>\n",
       "      <td> 0.438960</td>\n",
       "      <td> 0.326965</td>\n",
       "      <td> 0.455356</td>\n",
       "      <td> 0.478750</td>\n",
       "      <td> 0.595530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 0.672832</td>\n",
       "      <td> 0.612277</td>\n",
       "      <td> 0.658779</td>\n",
       "      <td> 0.658911</td>\n",
       "      <td> 0.557392</td>\n",
       "      <td> 0.560468</td>\n",
       "      <td> 0.570430</td>\n",
       "      <td> 0.647108</td>\n",
       "      <td> 0.567161</td>\n",
       "      <td> 0.583021</td>\n",
       "      <td> 0.645103</td>\n",
       "      <td> 0.554328</td>\n",
       "      <td> 0.729818</td>\n",
       "      <td> 0.448795</td>\n",
       "      <td> 0.410199</td>\n",
       "      <td> 0.518829</td>\n",
       "      <td> 0.639049</td>\n",
       "      <td> 0.647471</td>\n",
       "      <td> 0.452940</td>\n",
       "      <td> 0.492896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 0.521121</td>\n",
       "      <td> 0.567693</td>\n",
       "      <td> 0.616051</td>\n",
       "      <td> 0.494120</td>\n",
       "      <td> 0.589285</td>\n",
       "      <td> 0.550241</td>\n",
       "      <td> 0.574624</td>\n",
       "      <td> 0.517245</td>\n",
       "      <td> 0.518212</td>\n",
       "      <td> 0.444257</td>\n",
       "      <td> 0.531300</td>\n",
       "      <td> 0.551238</td>\n",
       "      <td> 0.607714</td>\n",
       "      <td> 0.555354</td>\n",
       "      <td> 0.424680</td>\n",
       "      <td> 0.571842</td>\n",
       "      <td> 0.452101</td>\n",
       "      <td> 0.329466</td>\n",
       "      <td> 0.677823</td>\n",
       "      <td> 0.455432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 0.495459</td>\n",
       "      <td> 0.487993</td>\n",
       "      <td> 0.557713</td>\n",
       "      <td> 0.588912</td>\n",
       "      <td> 0.515174</td>\n",
       "      <td> 0.435785</td>\n",
       "      <td> 0.469315</td>\n",
       "      <td> 0.558919</td>\n",
       "      <td> 0.576647</td>\n",
       "      <td> 0.402942</td>\n",
       "      <td> 0.545885</td>\n",
       "      <td> 0.571774</td>\n",
       "      <td> 0.488988</td>\n",
       "      <td> 0.308756</td>\n",
       "      <td> 0.525776</td>\n",
       "      <td> 0.551664</td>\n",
       "      <td> 0.551487</td>\n",
       "      <td> 0.445159</td>\n",
       "      <td> 0.466376</td>\n",
       "      <td> 0.492252</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              cor1      cor2      cor3      cor4      cor5      cor6  \\\n",
       "date                                                                   \n",
       "19480101  0.726977  0.652806  0.623012  0.506846  0.523203  0.457494   \n",
       "19480102  0.760688  0.604350  0.564686  0.636293  0.549962  0.649019   \n",
       "19480103  0.672832  0.612277  0.658779  0.658911  0.557392  0.560468   \n",
       "19480104  0.521121  0.567693  0.616051  0.494120  0.589285  0.550241   \n",
       "19480105  0.495459  0.487993  0.557713  0.588912  0.515174  0.435785   \n",
       "\n",
       "              cor7      cor8      cor9     cor10     cor11     cor12  \\\n",
       "date                                                                   \n",
       "19480101  0.426564  0.649643  0.608331  0.493753  0.451300  0.356393   \n",
       "19480102  0.635242  0.609948  0.445098  0.539979  0.508294  0.500434   \n",
       "19480103  0.570430  0.647108  0.567161  0.583021  0.645103  0.554328   \n",
       "19480104  0.574624  0.517245  0.518212  0.444257  0.531300  0.551238   \n",
       "19480105  0.469315  0.558919  0.576647  0.402942  0.545885  0.571774   \n",
       "\n",
       "             cor13     cor14     cor15     cor16     cor17     cor18  \\\n",
       "date                                                                   \n",
       "19480101  0.639460  0.383338  0.619769  0.575454  0.356389  0.587782   \n",
       "19480102  0.667818  0.454554  0.560843  0.438960  0.326965  0.455356   \n",
       "19480103  0.729818  0.448795  0.410199  0.518829  0.639049  0.647471   \n",
       "19480104  0.607714  0.555354  0.424680  0.571842  0.452101  0.329466   \n",
       "19480105  0.488988  0.308756  0.525776  0.551664  0.551487  0.445159   \n",
       "\n",
       "             cor19     cor20  \n",
       "date                          \n",
       "19480101  0.367948  0.546730  \n",
       "19480102  0.478750  0.595530  \n",
       "19480103  0.452940  0.492896  \n",
       "19480104  0.677823  0.455432  \n",
       "19480105  0.466376  0.492252  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfS.iloc[0:5, 2*num_analogues:3*num_analogues]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19480101  date.an1     19560126\n",
       "          date.an2     19850131\n",
       "          date.an3     20110114\n",
       "          date.an4     19850130\n",
       "          date.an5     19980114\n",
       "          date.an6     19551224\n",
       "          date.an7     19521221\n",
       "          date.an8     19681220\n",
       "          date.an9     19540119\n",
       "          date.an10    19870101\n",
       "          date.an11    19871224\n",
       "          date.an12    19610109\n",
       "          date.an13    20001211\n",
       "          date.an14    20010108\n",
       "          date.an15    20110112\n",
       "          date.an16    19560125\n",
       "          date.an17    19651223\n",
       "          date.an18    19651213\n",
       "          date.an19    20031209\n",
       "          date.an20    20110113\n",
       "19480102  date.an1     19681221\n",
       "          date.an2     20001211\n",
       "          date.an3     19551224\n",
       "          date.an4     20110112\n",
       "          date.an5     19560126\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#TRANSPOSE ANLG DATE COLS INTO SINGLE COL\n",
    "\n",
    "#Define temporary df containing only anlg dates cols\n",
    "df_anlg = dfS.iloc[:, 0:num_analogues]\n",
    "\n",
    "#remove index name before transpose (stack)\n",
    "df_anlg.index.name = \"\"\n",
    "\n",
    "#transpose (stack)into single col\n",
    "dateStack = df_anlg.stack()\n",
    "dateStack.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19480101  dis1     515.034\n",
       "          dis2     596.280\n",
       "          dis3     617.760\n",
       "          dis4     634.687\n",
       "          dis5     637.823\n",
       "          dis6     638.138\n",
       "          dis7     645.760\n",
       "          dis8     647.538\n",
       "          dis9     648.427\n",
       "          dis10    650.624\n",
       "          dis11    653.039\n",
       "          dis12    658.585\n",
       "          dis13    666.854\n",
       "          dis14    667.827\n",
       "          dis15    668.390\n",
       "          dis16    674.883\n",
       "          dis17    680.150\n",
       "          dis18    683.547\n",
       "          dis19    686.121\n",
       "          dis20    686.633\n",
       "19480102  dis1     692.016\n",
       "          dis2     699.794\n",
       "          dis3     712.922\n",
       "          dis4     742.944\n",
       "          dis5     755.533\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#TRANSPOSE DIS COLS INTO SINGLE COL\n",
    "\n",
    "#Define temporary df containing only dis cols\n",
    "df_dis = dfS.iloc[:, num_analogues:2*num_analogues]\n",
    "\n",
    "#remove index name before transpose (stack)\n",
    "df_dis.index.name = \"\"\n",
    "\n",
    "#transpose (stack) into single col and take absolute value\n",
    "disStack = df_dis.stack().abs()\n",
    "disStack.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19480101  cor1     0.726977\n",
       "          cor2     0.652806\n",
       "          cor3     0.623012\n",
       "          cor4     0.506846\n",
       "          cor5     0.523203\n",
       "          cor6     0.457494\n",
       "          cor7     0.426564\n",
       "          cor8     0.649643\n",
       "          cor9     0.608331\n",
       "          cor10    0.493753\n",
       "          cor11    0.451300\n",
       "          cor12    0.356393\n",
       "          cor13    0.639460\n",
       "          cor14    0.383338\n",
       "          cor15    0.619769\n",
       "          cor16    0.575454\n",
       "          cor17    0.356389\n",
       "          cor18    0.587782\n",
       "          cor19    0.367948\n",
       "          cor20    0.546730\n",
       "19480102  cor1     0.760688\n",
       "          cor2     0.604350\n",
       "          cor3     0.564686\n",
       "          cor4     0.636293\n",
       "          cor5     0.549962\n",
       "dtype: float64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#TRANSPOSE CORR COLS INTO SINGLE COL\n",
    "\n",
    "#Define temporary df containing only corr cols\n",
    "df_corr = dfS.iloc[:, 2*num_analogues:3*num_analogues]\n",
    "\n",
    "#remove index name before transpose (stack)\n",
    "df_corr.index.name = \"\"\n",
    "\n",
    "#transpose (stack) into single col\n",
    "corrStack = df_corr.stack()\n",
    "corrStack.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#BUILD NEW DF IN CORRECT FORMAT\n",
    "\n",
    "#CREATE DF OF CORRECT DIMENSIONS (n x num_cols)\n",
    "df_all = dfS.iloc[:, 0:num_cols]\n",
    "#rename cols\n",
    "df_all.columns = ['dateAnlg', 'Dis', 'Corr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dateAnlg</th>\n",
       "      <th>Dis</th>\n",
       "      <th>Corr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 20110114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 19551224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 19491206</td>\n",
       "      <td> 19741222</td>\n",
       "      <td> 19750110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19891221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 19971210</td>\n",
       "      <td> 19971209</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480106</th>\n",
       "      <td> 20020122</td>\n",
       "      <td> 19710120</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480107</th>\n",
       "      <td> 19781214</td>\n",
       "      <td> 19591227</td>\n",
       "      <td> 19510112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480108</th>\n",
       "      <td> 20040114</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19781214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480109</th>\n",
       "      <td> 20021230</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480110</th>\n",
       "      <td> 19550128</td>\n",
       "      <td> 19710116</td>\n",
       "      <td> 19800130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dateAnlg       Dis      Corr\n",
       "                                      \n",
       "19480101  19560126  19850131  20110114\n",
       "19480102  19681221  20001211  19551224\n",
       "19480103  19491206  19741222  19750110\n",
       "19480104  20031217  19921215  19891221\n",
       "19480105  19971210  19971209  19931217\n",
       "19480106  20020122  19710120  19931217\n",
       "19480107  19781214  19591227  19510112\n",
       "19480108  20040114  19591228  19781214\n",
       "19480109  20021230  20020121  19931217\n",
       "19480110  19550128  19710116  19800130"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#dfS.index.tolist()\n",
    "df_all.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dateAnlg</th>\n",
       "      <th>Dis</th>\n",
       "      <th>Corr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dateRef</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 19850131</td>\n",
       "      <td> 20110114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 20001211</td>\n",
       "      <td> 19551224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480103</th>\n",
       "      <td> 19491206</td>\n",
       "      <td> 19741222</td>\n",
       "      <td> 19750110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480104</th>\n",
       "      <td> 20031217</td>\n",
       "      <td> 19921215</td>\n",
       "      <td> 19891221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480105</th>\n",
       "      <td> 19971210</td>\n",
       "      <td> 19971209</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480106</th>\n",
       "      <td> 20020122</td>\n",
       "      <td> 19710120</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480107</th>\n",
       "      <td> 19781214</td>\n",
       "      <td> 19591227</td>\n",
       "      <td> 19510112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480108</th>\n",
       "      <td> 20040114</td>\n",
       "      <td> 19591228</td>\n",
       "      <td> 19781214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480109</th>\n",
       "      <td> 20021230</td>\n",
       "      <td> 20020121</td>\n",
       "      <td> 19931217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480110</th>\n",
       "      <td> 19550128</td>\n",
       "      <td> 19710116</td>\n",
       "      <td> 19800130</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dateAnlg       Dis      Corr\n",
       "dateRef                               \n",
       "19480101  19560126  19850131  20110114\n",
       "19480102  19681221  20001211  19551224\n",
       "19480103  19491206  19741222  19750110\n",
       "19480104  20031217  19921215  19891221\n",
       "19480105  19971210  19971209  19931217\n",
       "19480106  20020122  19710120  19931217\n",
       "19480107  19781214  19591227  19510112\n",
       "19480108  20040114  19591228  19781214\n",
       "19480109  20021230  20020121  19931217\n",
       "19480110  19550128  19710116  19800130"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = df_all.set_index(dfS['dateRef'] )\n",
    "df_all.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#REPLICATE INDEX COL 20 TIMES PER ROW\n",
    "\n",
    "#http://stackoverflow.com/questions/26777832/replicating-rows-in-a-pandas-data-frame-by-a-column-value\n",
    "#df2 = df.loc[np.repeat(df.index.values,df.n)]\n",
    "df_all = df_all.loc[np.repeat(df_all.index.values,num_analogues)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dateAnlg</th>\n",
       "      <th>Dis</th>\n",
       "      <th>Corr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dateRef</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 515.034</td>\n",
       "      <td> 0.726977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19850131</td>\n",
       "      <td> 596.280</td>\n",
       "      <td> 0.652806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20110114</td>\n",
       "      <td> 617.760</td>\n",
       "      <td> 0.623012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19850130</td>\n",
       "      <td> 634.687</td>\n",
       "      <td> 0.506846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19980114</td>\n",
       "      <td> 637.823</td>\n",
       "      <td> 0.523203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19551224</td>\n",
       "      <td> 638.138</td>\n",
       "      <td> 0.457494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19521221</td>\n",
       "      <td> 645.760</td>\n",
       "      <td> 0.426564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19681220</td>\n",
       "      <td> 647.538</td>\n",
       "      <td> 0.649643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19540119</td>\n",
       "      <td> 648.427</td>\n",
       "      <td> 0.608331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19870101</td>\n",
       "      <td> 650.624</td>\n",
       "      <td> 0.493753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19871224</td>\n",
       "      <td> 653.039</td>\n",
       "      <td> 0.451300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19610109</td>\n",
       "      <td> 658.585</td>\n",
       "      <td> 0.356393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20001211</td>\n",
       "      <td> 666.854</td>\n",
       "      <td> 0.639460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20010108</td>\n",
       "      <td> 667.827</td>\n",
       "      <td> 0.383338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20110112</td>\n",
       "      <td> 668.390</td>\n",
       "      <td> 0.619769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19560125</td>\n",
       "      <td> 674.883</td>\n",
       "      <td> 0.575454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19651223</td>\n",
       "      <td> 680.150</td>\n",
       "      <td> 0.356389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 19651213</td>\n",
       "      <td> 683.547</td>\n",
       "      <td> 0.587782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20031209</td>\n",
       "      <td> 686.121</td>\n",
       "      <td> 0.367948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480101</th>\n",
       "      <td> 20110113</td>\n",
       "      <td> 686.633</td>\n",
       "      <td> 0.546730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19681221</td>\n",
       "      <td> 692.016</td>\n",
       "      <td> 0.760688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 20001211</td>\n",
       "      <td> 699.794</td>\n",
       "      <td> 0.604350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19551224</td>\n",
       "      <td> 712.922</td>\n",
       "      <td> 0.564686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 20110112</td>\n",
       "      <td> 742.944</td>\n",
       "      <td> 0.636293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19480102</th>\n",
       "      <td> 19560126</td>\n",
       "      <td> 755.533</td>\n",
       "      <td> 0.549962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dateAnlg      Dis      Corr\n",
       "dateRef                              \n",
       "19480101  19560126  515.034  0.726977\n",
       "19480101  19850131  596.280  0.652806\n",
       "19480101  20110114  617.760  0.623012\n",
       "19480101  19850130  634.687  0.506846\n",
       "19480101  19980114  637.823  0.523203\n",
       "19480101  19551224  638.138  0.457494\n",
       "19480101  19521221  645.760  0.426564\n",
       "19480101  19681220  647.538  0.649643\n",
       "19480101  19540119  648.427  0.608331\n",
       "19480101  19870101  650.624  0.493753\n",
       "19480101  19871224  653.039  0.451300\n",
       "19480101  19610109  658.585  0.356393\n",
       "19480101  20001211  666.854  0.639460\n",
       "19480101  20010108  667.827  0.383338\n",
       "19480101  20110112  668.390  0.619769\n",
       "19480101  19560125  674.883  0.575454\n",
       "19480101  19651223  680.150  0.356389\n",
       "19480101  19651213  683.547  0.587782\n",
       "19480101  20031209  686.121  0.367948\n",
       "19480101  20110113  686.633  0.546730\n",
       "19480102  19681221  692.016  0.760688\n",
       "19480102  20001211  699.794  0.604350\n",
       "19480102  19551224  712.922  0.564686\n",
       "19480102  20110112  742.944  0.636293\n",
       "19480102  19560126  755.533  0.549962"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#REPLACE COLS WITH STACKED ANLG DATES, DIS AND CORR\n",
    "df_all.iloc[:,0] = list(dateStack)\n",
    "df_all.iloc[:,1] = list(disStack)\n",
    "df_all.iloc[:,2] = list(corrStack)\n",
    "df_all.head(25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(502040, 3)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dateAnlg</th>\n",
       "      <th>Dis</th>\n",
       "      <th>Corr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dateRef</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 20010823</td>\n",
       "      <td> 509.074</td>\n",
       "      <td> 0.463277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19990912</td>\n",
       "      <td> 512.342</td>\n",
       "      <td> 0.539685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 20020822</td>\n",
       "      <td> 516.161</td>\n",
       "      <td> 0.335925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 20130904</td>\n",
       "      <td> 516.773</td>\n",
       "      <td> 0.528130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19790906</td>\n",
       "      <td> 517.020</td>\n",
       "      <td> 0.467019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19490905</td>\n",
       "      <td> 517.301</td>\n",
       "      <td> 0.469278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19810911</td>\n",
       "      <td> 519.320</td>\n",
       "      <td> 0.547674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19960823</td>\n",
       "      <td> 520.936</td>\n",
       "      <td> 0.493447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19990911</td>\n",
       "      <td> 521.202</td>\n",
       "      <td> 0.568598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19590929</td>\n",
       "      <td> 526.219</td>\n",
       "      <td> 0.621988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19790905</td>\n",
       "      <td> 532.691</td>\n",
       "      <td> 0.470806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19920919</td>\n",
       "      <td> 532.711</td>\n",
       "      <td> 0.484619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19920918</td>\n",
       "      <td> 533.790</td>\n",
       "      <td> 0.415522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 19990915</td>\n",
       "      <td> 536.654</td>\n",
       "      <td> 0.610709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20160921</th>\n",
       "      <td> 20010829</td>\n",
       "      <td> 539.057</td>\n",
       "      <td> 0.456672</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dateAnlg      Dis      Corr\n",
       "dateRef                              \n",
       "20160921  20010823  509.074  0.463277\n",
       "20160921  19990912  512.342  0.539685\n",
       "20160921  20020822  516.161  0.335925\n",
       "20160921  20130904  516.773  0.528130\n",
       "20160921  19790906  517.020  0.467019\n",
       "20160921  19490905  517.301  0.469278\n",
       "20160921  19810911  519.320  0.547674\n",
       "20160921  19960823  520.936  0.493447\n",
       "20160921  19990911  521.202  0.568598\n",
       "20160921  19590929  526.219  0.621988\n",
       "20160921  19790905  532.691  0.470806\n",
       "20160921  19920919  532.711  0.484619\n",
       "20160921  19920918  533.790  0.415522\n",
       "20160921  19990915  536.654  0.610709\n",
       "20160921  20010829  539.057  0.456672"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.tail(15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#dfS_select = df_all[df_all.index <= 19600101]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#SAVE TO CSV FILE\n",
    "\n",
    "#df_all.to_csv('analogues_19480101_20151225.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test2.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test_gt1yr.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test_gt1yr_birdhouse.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test_11anlgs.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test_span2months.json', float_format='%.7f', sep='\\t')\n",
    "#df_all.to_csv('test_gt1yr_birdhouse_03Jan2013.json', float_format='%.7f', sep='\\t')\n",
    "\n",
    "#df_all.to_csv('analogues_19480101_20160520.json', float_format='%.7f', sep='\\t')\n",
    "\n",
    "#dfS_select.to_csv('analogues_select.json', float_format='%.7f', sep='\\t')\n",
    "\n",
    "df_all.to_csv('ana_slp_surface_base_rms_NA_sim_1948-01-01_2016-09-23_base_1948-01-01_2016-09-23_-80.0_50.0_22.5_70.0_1_30_20.json', float_format='%.7f', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/homel/cnangini/PROJECTS/analogues'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0773818303236 0.952985818969\n"
     ]
    }
   ],
   "source": [
    "print df_all['Corr'].min(), df_all['Corr'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "284.431297359 1369.41118104\n"
     ]
    }
   ],
   "source": [
    "print df_all['Dis'].min(), df_all['Dis'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#check 1970-1979\n",
    "df_check =  df_all[ (df_all['dateAnlg'] >= 19700101) & (df_all['dateAnlg'] <= 19791231) ]\n",
    "\n",
    "\n",
    "#((df['column1']=='banana') & (df['colour']=='green'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(75935, 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_check.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_check =  df_all[ (df_all['dateAnlg'] >= 20100101) & (df_all['dateAnlg'] <= 20191231) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(44554, 3)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_check.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#cut file for testing purposes\n",
    "df_select = df_all.iloc[19000:19100,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 3)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_select.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dateAnlg</th>\n",
       "      <th>Dis</th>\n",
       "      <th>Corr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dateRef</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19840828</td>\n",
       "      <td>449.988840</td>\n",
       "      <td>0.546204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19590909</td>\n",
       "      <td>453.044659</td>\n",
       "      <td>0.612506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>20030819</td>\n",
       "      <td>453.638210</td>\n",
       "      <td>0.559760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19970801</td>\n",
       "      <td>457.039066</td>\n",
       "      <td>0.597291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19980815</td>\n",
       "      <td>459.252587</td>\n",
       "      <td>0.534387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19990829</td>\n",
       "      <td>466.308358</td>\n",
       "      <td>0.580896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19510820</td>\n",
       "      <td>466.860763</td>\n",
       "      <td>0.727749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>20130816</td>\n",
       "      <td>467.150673</td>\n",
       "      <td>0.623782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19990717</td>\n",
       "      <td>471.338490</td>\n",
       "      <td>0.571134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19500812</th>\n",
       "      <td>19790816</td>\n",
       "      <td>471.416637</td>\n",
       "      <td>0.661347</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          dateAnlg         Dis      Corr\n",
       "dateRef                                 \n",
       "19500812  19840828  449.988840  0.546204\n",
       "19500812  19590909  453.044659  0.612506\n",
       "19500812  20030819  453.638210  0.559760\n",
       "19500812  19970801  457.039066  0.597291\n",
       "19500812  19980815  459.252587  0.534387\n",
       "19500812  19990829  466.308358  0.580896\n",
       "19500812  19510820  466.860763  0.727749\n",
       "19500812  20130816  467.150673  0.623782\n",
       "19500812  19990717  471.338490  0.571134\n",
       "19500812  19790816  471.416637  0.661347"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_select.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_chunk = df_all.iloc[149000:149100,:]\n",
    "df_chunk2 = df_all.iloc[106000:106100,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df_select = df_select.append(df_chunk)\n",
    "df_select = df_select.append(df_chunk2)\n",
    "#df_select = pd.concat([df_select,df_chunk])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(300, 3)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_select.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#SAVE SELECTION TO TSV FILE\n",
    "\n",
    "#df_select.to_csv('analogues_reformat_all_select.tsv', float_format='%.2f', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0773818303236 0.952985818969\n"
     ]
    }
   ],
   "source": [
    "print df_all['Corr'].min(), df_all['Corr'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "284.431297359 1369.41118104\n"
     ]
    }
   ],
   "source": [
    "print df_all['Dis'].min(), df_all['Dis'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.124791552993 0.821019151951\n"
     ]
    }
   ],
   "source": [
    "df_select['Corr'].max()\n",
    "df_select['Corr'].min()\n",
    "print df_select['Corr'].min(), df_select['Corr'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "350.217929418 696.281040107\n"
     ]
    }
   ],
   "source": [
    "df_select['Dis'].max()\n",
    "df_select['Dis'].min()\n",
    "print df_select['Dis'].min(), df_select['Dis'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}