examples-read-write.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import importlib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import glob\n",
    "\n",
    "import d6tstack.combine_csv\n",
    "from d6tstack.utils import PrintLogger\n",
    "logger = PrintLogger()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CombinerCSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['test-data/input/test-data-input-csv-colmismatch-mar.csv', 'test-data/input/test-data-input-csv-colmismatch-feb.csv', 'test-data/input/test-data-input-csv-colmismatch-jan.csv']\n"
     ]
    }
   ],
   "source": [
    "cfg_fnames = list(glob.glob('test-data/input/test-data-input-csv-colmismatch-*.csv'))\n",
    "print(cfg_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = d6tstack.combine_csv.CombinerCSV(cfg_fnames, all_strings=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-02</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-03</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-04</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-05</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cost        date  profit  profit2  sales  \\\n",
       "0   -80  2011-01-01      20      NaN    100   \n",
       "1   -80  2011-01-02      20      NaN    100   \n",
       "2   -80  2011-01-03      20      NaN    100   \n",
       "3   -80  2011-01-04      20      NaN    100   \n",
       "4   -80  2011-01-05      20      NaN    100   \n",
       "\n",
       "                                          filename  \n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv  "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.to_csv(output_dir='test-data/output/',overwrite=True)\n",
    "pd.read_csv('test-data/output/test-data-input-csv-colmismatch-jan-matched.csv').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-02</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-03</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-04</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-05</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cost        date  profit  profit2  sales  \\\n",
       "0   -80  2011-01-01      20      NaN    100   \n",
       "1   -80  2011-01-02      20      NaN    100   \n",
       "2   -80  2011-01-03      20      NaN    100   \n",
       "3   -80  2011-01-04      20      NaN    100   \n",
       "4   -80  2011-01-05      20      NaN    100   \n",
       "\n",
       "                                          filename  \n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv  "
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# doesn't raise any warnings... thought we had overwrite warnings implemented?\n",
    "c.to_csv(output_dir='test-data/output/',overwrite=False)\n",
    "pd.read_csv('test-data/output/test-data-input-csv-colmismatch-jan-matched.csv').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "      <th>filename</th>\n",
       "      <th>filename.1</th>\n",
       "      <th>filename.2</th>\n",
       "      <th>filename.3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-02</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-03</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-04</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-05</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan-matched.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cost        date  profit  profit2  sales  \\\n",
       "0   -80  2011-01-01      20      NaN    100   \n",
       "1   -80  2011-01-02      20      NaN    100   \n",
       "2   -80  2011-01-03      20      NaN    100   \n",
       "3   -80  2011-01-04      20      NaN    100   \n",
       "4   -80  2011-01-05      20      NaN    100   \n",
       "\n",
       "                                          filename  \\\n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "\n",
       "                                        filename.1  \\\n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "\n",
       "                                        filename.2  \\\n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv   \n",
       "\n",
       "                                        filename.3  \n",
       "0  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "1  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "2  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "3  test-data-input-csv-colmismatch-jan-matched.csv  \n",
       "4  test-data-input-csv-colmismatch-jan-matched.csv  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# adds 3 columns for filename\n",
    "c.to_csv(output_dir='test-data/output/',overwrite=True)\n",
    "pd.read_csv('test-data/output/test-data-input-csv-colmismatch-jan-matched.csv').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# not writing a file / raising error\n",
    "c.to_csv(output_dir='test-data/output/',separate_files=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# not writing a file\n",
    "c.to_csv(output_dir='test-data/output/',separate_files=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py:271: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  df_all = pd.concat(dfl_all)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>filename</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-01</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400.0</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-02</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400.0</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-03</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400.0</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-04</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400.0</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-05</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400.0</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cost        date                                 filename  profit  profit2  \\\n",
       "0  -100  2011-03-01  test-data-input-csv-colmismatch-mar.csv     200    400.0   \n",
       "1  -100  2011-03-02  test-data-input-csv-colmismatch-mar.csv     200    400.0   \n",
       "2  -100  2011-03-03  test-data-input-csv-colmismatch-mar.csv     200    400.0   \n",
       "3  -100  2011-03-04  test-data-input-csv-colmismatch-mar.csv     200    400.0   \n",
       "4  -100  2011-03-05  test-data-input-csv-colmismatch-mar.csv     200    400.0   \n",
       "\n",
       "   sales  \n",
       "0    300  \n",
       "1    300  \n",
       "2    300  \n",
       "3    300  \n",
       "4    300  "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ok\n",
    "c.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=False)\n",
    "pd.read_csv('test-data/output/test-combined.csv').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>filename</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-06</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-07</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-08</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-09</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-10</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    cost        date                                 filename  profit  \\\n",
       "25   -80  2011-01-06  test-data-input-csv-colmismatch-jan.csv      20   \n",
       "26   -80  2011-01-07  test-data-input-csv-colmismatch-jan.csv      20   \n",
       "27   -80  2011-01-08  test-data-input-csv-colmismatch-jan.csv      20   \n",
       "28   -80  2011-01-09  test-data-input-csv-colmismatch-jan.csv      20   \n",
       "29   -80  2011-01-10  test-data-input-csv-colmismatch-jan.csv      20   \n",
       "\n",
       "    profit2  sales  \n",
       "25      NaN    100  \n",
       "26      NaN    100  \n",
       "27      NaN    100  \n",
       "28      NaN    100  \n",
       "29      NaN    100  "
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('test-data/output/test-combined.csv').tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "to_csv() got an unexpected keyword argument 'is_col_common'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-95-436aa482cf5f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# add is_col_common to pass through\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/test-combined.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseparate_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mis_col_common\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: to_csv() got an unexpected keyword argument 'is_col_common'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# add is_col_common to pass through\n",
    "c.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=False,is_col_common=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "to_csv() got an unexpected keyword argument 'streaming'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-96-970c7618acf7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# how do I do streaming?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/test-combined.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseparate_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstreaming\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m: to_csv() got an unexpected keyword argument 'streaming'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# how do I do streaming?\n",
    "c.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=False,streaming=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'CombinerCSV' object has no attribute 'to_parquet'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-97-bb0939e7210f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'CombinerCSV' object has no attribute 'to_parquet'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "c.to_parquet(output_dir='test-data/output/')\n",
    "import pyarrow.parquet as pq\n",
    "table = pq.read_table('test-data/output/test-data-input-csv-colmismatch-jan-matched')\n",
    "table.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py:271: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  df_all = pd.concat(dfl_all)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.to_sql('mysql+mysqlconnector://testusr:testusr@localhost/test','testd6tstack')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sqlalchemy.engine import create_engine\n",
    "sqlcnxn = create_engine('mysql+mysqlconnector://testusr:testusr@localhost/test').connect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>filename</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-01</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-02</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-03</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-04</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>-100</td>\n",
       "      <td>2011-03-05</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "      <td>200</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  cost        date                                 filename profit  \\\n",
       "0      0  -100  2011-03-01  test-data-input-csv-colmismatch-mar.csv    200   \n",
       "1      1  -100  2011-03-02  test-data-input-csv-colmismatch-mar.csv    200   \n",
       "2      2  -100  2011-03-03  test-data-input-csv-colmismatch-mar.csv    200   \n",
       "3      3  -100  2011-03-04  test-data-input-csv-colmismatch-mar.csv    200   \n",
       "4      4  -100  2011-03-05  test-data-input-csv-colmismatch-mar.csv    200   \n",
       "\n",
       "  profit2 sales  \n",
       "0     400   300  \n",
       "1     400   300  \n",
       "2     400   300  \n",
       "3     400   300  \n",
       "4     400   300  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_sql_table('testd6tstack',sqlcnxn).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>cost</th>\n",
       "      <th>date</th>\n",
       "      <th>filename</th>\n",
       "      <th>profit</th>\n",
       "      <th>profit2</th>\n",
       "      <th>sales</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>5</td>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-06</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>6</td>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-07</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>7</td>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-08</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>8</td>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-09</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>9</td>\n",
       "      <td>-80</td>\n",
       "      <td>2011-01-10</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "      <td>20</td>\n",
       "      <td>None</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    index cost        date                                 filename profit  \\\n",
       "25      5  -80  2011-01-06  test-data-input-csv-colmismatch-jan.csv     20   \n",
       "26      6  -80  2011-01-07  test-data-input-csv-colmismatch-jan.csv     20   \n",
       "27      7  -80  2011-01-08  test-data-input-csv-colmismatch-jan.csv     20   \n",
       "28      8  -80  2011-01-09  test-data-input-csv-colmismatch-jan.csv     20   \n",
       "29      9  -80  2011-01-10  test-data-input-csv-colmismatch-jan.csv     20   \n",
       "\n",
       "   profit2 sales  \n",
       "25    None   100  \n",
       "26    None   100  \n",
       "27    None   100  \n",
       "28    None   100  \n",
       "29    None   100  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_sql_table('testd6tstack',sqlcnxn).tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CombinerCSVAdvanced.to_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>profit3</th>\n",
       "      <th>sales</th>\n",
       "      <th>cost</th>\n",
       "      <th>profit</th>\n",
       "      <th>filename</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-03-01</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "      <td>-100</td>\n",
       "      <td>200</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-03-02</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "      <td>-100</td>\n",
       "      <td>200</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011-03-03</td>\n",
       "      <td>400</td>\n",
       "      <td>300</td>\n",
       "      <td>-100</td>\n",
       "      <td>200</td>\n",
       "      <td>test-data-input-csv-colmismatch-mar.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-02-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>-90</td>\n",
       "      <td>110</td>\n",
       "      <td>test-data-input-csv-colmismatch-feb.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-02-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>-90</td>\n",
       "      <td>110</td>\n",
       "      <td>test-data-input-csv-colmismatch-feb.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011-02-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>200</td>\n",
       "      <td>-90</td>\n",
       "      <td>110</td>\n",
       "      <td>test-data-input-csv-colmismatch-feb.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2011-01-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>-80</td>\n",
       "      <td>20</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2011-01-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>-80</td>\n",
       "      <td>20</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2011-01-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>100</td>\n",
       "      <td>-80</td>\n",
       "      <td>20</td>\n",
       "      <td>test-data-input-csv-colmismatch-jan.csv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date profit3 sales  cost profit  \\\n",
       "0  2011-03-01     400   300  -100    200   \n",
       "1  2011-03-02     400   300  -100    200   \n",
       "2  2011-03-03     400   300  -100    200   \n",
       "0  2011-02-01     NaN   200   -90    110   \n",
       "1  2011-02-02     NaN   200   -90    110   \n",
       "2  2011-02-03     NaN   200   -90    110   \n",
       "0  2011-01-01     NaN   100   -80     20   \n",
       "1  2011-01-02     NaN   100   -80     20   \n",
       "2  2011-01-03     NaN   100   -80     20   \n",
       "\n",
       "                                  filename  \n",
       "0  test-data-input-csv-colmismatch-mar.csv  \n",
       "1  test-data-input-csv-colmismatch-mar.csv  \n",
       "2  test-data-input-csv-colmismatch-mar.csv  \n",
       "0  test-data-input-csv-colmismatch-feb.csv  \n",
       "1  test-data-input-csv-colmismatch-feb.csv  \n",
       "2  test-data-input-csv-colmismatch-feb.csv  \n",
       "0  test-data-input-csv-colmismatch-jan.csv  \n",
       "1  test-data-input-csv-colmismatch-jan.csv  \n",
       "2  test-data-input-csv-colmismatch-jan.csv  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combiner2 = d6tstack.combine_csv.CombinerCSVAdvanced(c, c.preview_columns()['columns_all'], {'profit2':'profit3'})\n",
    "combiner2.preview_combine() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "combine_save() got an unexpected keyword argument 'parquet_output'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-53-998d023decfd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcombiner2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/test-combined.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseparate_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize)\u001b[0m\n\u001b[1;32m    617\u001b[0m         \"\"\"\n\u001b[1;32m    618\u001b[0m         convert_to_csv_parquet(self, out_filename=out_filename, separate_files=separate_files, output_dir=output_dir,\n\u001b[0;32m--> 619\u001b[0;31m                                suffix=suffix, overwrite=overwrite, streaming=streaming, chunksize=chunksize)\n\u001b[0m\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m     def to_parquet(self, out_filename=None, separate_files=True, output_dir=None, suffix='-matched',\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mconvert_to_csv_parquet\u001b[0;34m(combiner, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize, parquet_output)\u001b[0m\n\u001b[1;32m     58\u001b[0m                             chunksize=chunksize, parquet_output=parquet_output)\n\u001b[1;32m     59\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mstreaming\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mout_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m         \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparquet_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparquet_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mout_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: combine_save() got an unexpected keyword argument 'parquet_output'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# bug??\n",
    "combiner2.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "align_save() got an unexpected keyword argument 'parquet_output'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-54-ebe0679ac8cd>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# bug??\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mcombiner2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/test-combined.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseparate_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize)\u001b[0m\n\u001b[1;32m    617\u001b[0m         \"\"\"\n\u001b[1;32m    618\u001b[0m         convert_to_csv_parquet(self, out_filename=out_filename, separate_files=separate_files, output_dir=output_dir,\n\u001b[0;32m--> 619\u001b[0;31m                                suffix=suffix, overwrite=overwrite, streaming=streaming, chunksize=chunksize)\n\u001b[0m\u001b[1;32m    620\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m     def to_parquet(self, out_filename=None, separate_files=True, output_dir=None, suffix='-matched',\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mconvert_to_csv_parquet\u001b[0;34m(combiner, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize, parquet_output)\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mseparate_files\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         combiner.align_save(output_dir=output_dir, suffix=suffix, overwrite=overwrite,\n\u001b[0;32m---> 58\u001b[0;31m                             chunksize=chunksize, parquet_output=parquet_output)\n\u001b[0m\u001b[1;32m     59\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mstreaming\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mout_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparquet_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparquet_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: align_save() got an unexpected keyword argument 'parquet_output'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# bug??\n",
    "combiner2.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "align_save() got an unexpected keyword argument 'parquet_output'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-58-79723936bb0c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mcombiner2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_parquet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mto_parquet\u001b[0;34m(self, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize)\u001b[0m\n\u001b[1;32m    637\u001b[0m         convert_to_csv_parquet(self, out_filename=out_filename, separate_files=separate_files, output_dir=output_dir,\n\u001b[1;32m    638\u001b[0m                                \u001b[0msuffix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msuffix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moverwrite\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moverwrite\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstreaming\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstreaming\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 639\u001b[0;31m                                parquet_output=True)\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mconvert_to_csv_parquet\u001b[0;34m(combiner, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize, parquet_output)\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mseparate_files\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         combiner.align_save(output_dir=output_dir, suffix=suffix, overwrite=overwrite,\n\u001b[0;32m---> 58\u001b[0;31m                             chunksize=chunksize, parquet_output=parquet_output)\n\u001b[0m\u001b[1;32m     59\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mstreaming\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mout_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparquet_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparquet_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: align_save() got an unexpected keyword argument 'parquet_output'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "combiner2.to_parquet(output_dir='test-data/output/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEBUG: large files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "['/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20140401_20140430_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20140501_20140829_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20140901_20141231_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20150101_20150630_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20150701_20151231_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20160101_20160630_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20160701_20161230_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20170102_20170630_D.txt', '/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20170703_20170731_D.txt']\n"
     ]
    }
   ],
   "source": [
    "cfg_fnames = list(np.sort(glob.glob('/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_*.txt')))\n",
    "print(len(cfg_fnames))\n",
    "print(cfg_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "['/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20150101_20150630_D.txt']\n"
     ]
    }
   ],
   "source": [
    "cfg_fnames = list(np.sort(glob.glob('/mnt/data/dev/ubs-alphahack2017-shared/data-raw/ihs/US_Factors_Zscores/US_Factors_TotalCap_Cusip_Zscore_Historical_20150101_20150630_D.txt')))\n",
    "print(len(cfg_fnames))\n",
    "print(cfg_fnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = d6tstack.combine_csv.CombinerCSV(cfg_fnames, all_strings=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.is_all_equal()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-3dcf1763c8be>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# how do I do streaming?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'test-data/output/test-combined.csv'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mseparate_files\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mto_csv\u001b[0;34m(self, out_filename, separate_files, output_dir, suffix, overwrite, chunksize)\u001b[0m\n\u001b[1;32m    434\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    435\u001b[0m         convert_to_csv_parquet(self, out_filename=out_filename, separate_files=separate_files, output_dir=output_dir,\n\u001b[0;32m--> 436\u001b[0;31m                                suffix=suffix, overwrite=overwrite, streaming=False, chunksize=chunksize)\n\u001b[0m\u001b[1;32m    437\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    438\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mconvert_to_csv_parquet\u001b[0;34m(combiner, out_filename, separate_files, output_dir, suffix, overwrite, streaming, chunksize, parquet_output)\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_filename\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparquet_output\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparquet_output\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mout_filename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m         \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcombiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcombine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     63\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mparquet_output\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m             \u001b[0;32mimport\u001b[0m \u001b[0mpyarrow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mcombine\u001b[0;34m(self, is_col_common, is_preview)\u001b[0m\n\u001b[1;32m    261\u001b[0m         \"\"\"\n\u001b[1;32m    262\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 263\u001b[0;31m         \u001b[0mdfl_all\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'reading full file'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_preview\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mis_preview\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    264\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    265\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mread_csv_all\u001b[0;34m(self, msg, is_preview, chunksize, cfg_col_sel, cfg_col_rename)\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m' '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mntpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ok'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m             \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mis_preview\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mis_preview\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mcfg_col_sel\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mcfg_col_rename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m                 \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mapply_select_rename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg_col_sel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg_col_rename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(self, fname, is_preview, chunksize)\u001b[0m\n\u001b[1;32m    113\u001b[0m         \u001b[0mcfg_nrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnrows_preview\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mis_preview\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m         return pd.read_csv(fname, dtype=cfg_dype, nrows=cfg_nrows, chunksize=chunksize,\n\u001b[0;32m--> 115\u001b[0;31m                            **self.read_csv_params)\n\u001b[0m\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m     def read_csv_all(self, msg=None, is_preview=False, chunksize=None, cfg_col_sel=None,\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, doublequote, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    676\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    679\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    680\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    445\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 446\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    447\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1034\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'skipfooter not supported for iteration'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1036\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1037\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1846\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1847\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1849\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_integer_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m    809\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_integer_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    812\u001b[0m     \"\"\"\n\u001b[1;32m    813\u001b[0m     \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mprovided\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0man\u001b[0m \u001b[0minteger\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "# how do I do streaming? this loads everything into memeory...\n",
    "c.to_csv(out_filename='test-data/output/test-combined.csv',separate_files=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-75-6289d35d16bb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_sql_stream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'mysql+mysqlconnector://testusr:testusr@localhost/test'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'testd6tstack'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/data/dev/d6t-lib/d6tstack/d6tstack/combine_csv.py\u001b[0m in \u001b[0;36mto_sql_stream\u001b[0;34m(self, cnxn_string, table_name, if_exists, chunksize, sql_chunksize, cfg_col_sel, is_col_common, cfg_col_rename)\u001b[0m\n\u001b[1;32m    403\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    404\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_log\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'processing '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mntpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'ok'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 405\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mdf_chunk\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunksize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mchunksize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    406\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcfg_col_sel\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mcfg_col_rename\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    407\u001b[0m                     \u001b[0mdf_chunk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mapply_select_rename\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_chunk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg_col_sel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcfg_col_rename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1006\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1007\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_chunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1008\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1009\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mget_chunk\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m   1068\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1069\u001b[0m             \u001b[0msize\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnrows\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_currow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1070\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1071\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1072\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1034\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'skipfooter not supported for iteration'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1035\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1036\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1037\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1038\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1846\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1847\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1849\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_column_data\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_tokens\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._convert_with_dtype\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_integer_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m    809\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 811\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mis_integer_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    812\u001b[0m     \"\"\"\n\u001b[1;32m    813\u001b[0m     \u001b[0mCheck\u001b[0m \u001b[0mwhether\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mprovided\u001b[0m \u001b[0marray\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mof\u001b[0m \u001b[0man\u001b[0m \u001b[0minteger\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "c.to_sql_stream('mysql+mysqlconnector://testusr:testusr@localhost/test','testd6tstack')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}