test_large_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing large csv table in compressed format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## read in chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checked externally, that the largest number of species id is 117,582,065 and the largest of fishid (named as wdpaid) 5,090,435. Therefore a `np.uint32` (0 to 4,294,967,295, four bytes) should be sufficient to hold the numbers. This reduces the default memory consumption by half (default is `np.int64`, which takes four bytes, range 0 to 18,446,744,073,709,551,615)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "int_types = [\"uint8\", \"int8\", \"int16\", \"uint16\", \"uint32\", \"int32\", 'int64']\n",
    "for it in int_types:\n",
    "    print(np.iinfo(it))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The species lookup table contains information not needed. Trim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sis = pd.read_csv(\"sis_2017.csv\")\n",
    "\n",
    "columns=['id_no', 'kingdom', 'phylum', 'class', 'order_', 'family', 'genus', 'code']\n",
    "sis_clean = pd.DataFrame(sis[columns])\n",
    "\n",
    "sis_clean = sis_clean.astype(dtype={\"id_no\": \"uint32\",\n",
    "                                    'kingdom': \"category\",\n",
    "                                    'phylum': \"category\",\n",
    "                                     'class': \"category\",\n",
    "                                     'order_': \"category\",\n",
    "                                     'family': \"category\",\n",
    "                                     'genus': \"category\",\n",
    "                                     'code': \"category\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the `id_no` as the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sis_clean_index = sis_clean.set_index('id_no')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load species in chunks, testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "species_2 = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=40000, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b1 = species_2.get_chunk()\n",
    "b2 = species_2.get_chunk()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merging two dataframes performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit b1.merge(sis_clean, left_on='ID_NO', right_on='id_no')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use index join performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit b1.join(sis_clean_index, how='inner', on='ID_NO')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It appears that the index join method seems slightly quicker - proceed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def analyse_chunk(df_chunk, species_attributes=sis_clean_index):\n",
    "#     # join species attributes, which has id_no as index and filtering\n",
    "#     df_chunk = df_chunk.join(species_attributes, how='inner', on='ID_NO')\n",
    "#     # pivot and return\n",
    "#     return df_chunk.pivot_table(index='WDPAID', columns='class', values='ID_NO', fill_value=0, aggfunc=np.count_nonzero)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`isin()` seems very fast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit b1[b1.ID_NO.isin(sis_clean_index.index)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%timeit b1.groupby('WDPAID').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bb = b1.groupby('WDPAID').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single machine version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_chunk_mk2(df_chunk, species_filter):\n",
    "    # find relevent species entries using the species_filter, which has id_no as index and filtering\n",
    "    df_chunk = df_chunk[df_chunk.ID_NO.isin(species_filter.index)]\n",
    "    # group and simple count\n",
    "    return df_chunk.groupby('WDPAID').count()\n",
    "\n",
    "def concat_chunk_results(df_chunks):\n",
    "    # list of chunks to concat, \n",
    "    # more effcient to have a couple of dfs, than appending each df and sum\n",
    "    df_chunk = pd.concat(df_chunks)\n",
    "    return df_chunk.groupby('WDPAID').sum()\n",
    "\n",
    "def chunk_worker(df_chunk, chunksize=40000):\n",
    "    counter = 0\n",
    "    \n",
    "    result = []\n",
    "    result.append(analyse_chunk_mk2(df_chunk))\n",
    "        \n",
    "    if counter % 10 == 0:\n",
    "        print('Processed rows:{:,}'.format(counter*chunksize))\n",
    "        result = [concat_chunk_results(result)]\n",
    "        \n",
    "    # ensure no un\n",
    "    return concat_chunk_results(result)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the `id_no` as the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_main():\n",
    "    import time\n",
    "    from datetime import timedelta\n",
    "    start_time = time.time()\n",
    "    chunksize = 4000000\n",
    "    \n",
    "    # read species attributes\n",
    "    sis = pd.read_csv(\"sis_2017.csv\")\n",
    "\n",
    "    columns=['id_no', 'kingdom', 'phylum', 'class', 'order_', 'family', 'genus', 'code']\n",
    "    sis_clean = pd.DataFrame(sis[columns])\n",
    "    sis_clean = sis_clean.astype(dtype={\"id_no\": \"uint32\",\n",
    "                                        'kingdom': \"category\",\n",
    "                                        'phylum': \"category\",\n",
    "                                         'class': \"category\",\n",
    "                                         'order_': \"category\",\n",
    "                                         'family': \"category\",\n",
    "                                         'genus': \"category\",\n",
    "                                         'code': \"category\"})\n",
    "    \n",
    "    sis_clean_index = sis_clean.set_index('id_no')\n",
    "\n",
    "    # read result in chunk\n",
    "    species = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=chunksize, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})\n",
    "    \n",
    "    # run\n",
    "    result = []\n",
    "    total_rows = 0\n",
    "    counter = 0\n",
    "    \n",
    "    for df_chunk in species:\n",
    "        \n",
    "#         if counter == 101:\n",
    "#             break\n",
    "               \n",
    "        result.append(analyse_chunk_mk2(df_chunk, sis_clean_index))\n",
    "        \n",
    "        if counter % 10 == 0:\n",
    "            print('Processed rows: {:,} '.format(counter*chunksize))\n",
    "            result = [concat_chunk_results(result)]\n",
    "        \n",
    "        if counter % 100 == 0:\n",
    "            elapsed = time.time() - start_time\n",
    "            print('Elapsed time: {}'.format(str(timedelta(seconds=elapsed))))\n",
    "            print('Memory usage of the output: {:,.1f} KB \\n---'.format(result[0].memory_usage().sum()/1024))\n",
    "            \n",
    "        counter += 1\n",
    "        \n",
    "    # final push\n",
    "    result = concat_chunk_results(result)\n",
    "    result.to_csv('output.csv')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed rows: 0 \n",
      "Elapsed time: 0:00:03.385200\n",
      "Memory usage of the output: 26,681.4 KB \n",
      "---\n",
      "Processed rows: 40,000,000 \n",
      "Processed rows: 80,000,000 \n",
      "Processed rows: 120,000,000 \n",
      "Processed rows: 160,000,000 \n",
      "Processed rows: 200,000,000 \n",
      "Processed rows: 240,000,000 \n",
      "Processed rows: 280,000,000 \n",
      "Processed rows: 320,000,000 \n",
      "Processed rows: 360,000,000 \n",
      "Processed rows: 400,000,000 \n",
      "Elapsed time: 0:04:19.864800\n",
      "Memory usage of the output: 77,621.3 KB \n",
      "---\n"
     ]
    }
   ],
   "source": [
    "# output = test_main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed rows: 0 \n",
      "Elapsed time: 0:00:03.260400\n",
      "Memory usage of the output: 26,681.4 KB \n",
      "---\n",
      "Processed rows: 40,000,000 \n",
      "Processed rows: 80,000,000 \n",
      "Processed rows: 120,000,000 \n",
      "Processed rows: 160,000,000 \n",
      "Processed rows: 200,000,000 \n",
      "Processed rows: 240,000,000 \n",
      "Processed rows: 280,000,000 \n",
      "Processed rows: 320,000,000 \n",
      "Processed rows: 360,000,000 \n",
      "Processed rows: 400,000,000 \n",
      "Elapsed time: 0:04:23.810600\n",
      "Memory usage of the output: 77,621.3 KB \n",
      "---\n",
      "Processed rows: 440,000,000 \n",
      "Processed rows: 480,000,000 \n",
      "Processed rows: 520,000,000 \n",
      "Processed rows: 560,000,000 \n",
      "Processed rows: 600,000,000 \n",
      "Processed rows: 640,000,000 \n",
      "Processed rows: 680,000,000 \n",
      "Processed rows: 720,000,000 \n",
      "Processed rows: 760,000,000 \n",
      "Processed rows: 800,000,000 \n",
      "Elapsed time: 0:08:45.023200\n",
      "Memory usage of the output: 77,817.9 KB \n",
      "---\n",
      "Processed rows: 840,000,000 \n",
      "Processed rows: 880,000,000 \n",
      "Processed rows: 920,000,000 \n",
      "Processed rows: 960,000,000 \n",
      "Processed rows: 1,000,000,000 \n",
      "Processed rows: 1,040,000,000 \n",
      "Processed rows: 1,080,000,000 \n",
      "Processed rows: 1,120,000,000 \n",
      "Processed rows: 1,160,000,000 \n",
      "Processed rows: 1,200,000,000 \n",
      "Elapsed time: 0:13:13.768400\n",
      "Memory usage of the output: 77,841.8 KB \n",
      "---\n",
      "Processed rows: 1,240,000,000 \n",
      "Processed rows: 1,280,000,000 \n",
      "Processed rows: 1,320,000,000 \n",
      "Processed rows: 1,360,000,000 \n",
      "Processed rows: 1,400,000,000 \n"
     ]
    }
   ],
   "source": [
    "# output = test_main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parallel computing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "def worker(df_chunk):\n",
    "    # find relevent species entries using the species_filter, which has id_no as index and filtering\n",
    "    df_chunk = df_chunk[df_chunk.ID_NO.isin(sis_clean_index.index)]\n",
    "    # group and simple count\n",
    "    return df_chunk.groupby('WDPAID').count()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_main_parallel():\n",
    "    import time\n",
    "    from datetime import timedelta\n",
    "    start_time = time.time()\n",
    "    chunksize = 8000000\n",
    "    \n",
    "    # read species attributes\n",
    "    sis = pd.read_csv(\"sis_2017.csv\")\n",
    "\n",
    "    columns=['id_no', 'kingdom', 'phylum', 'class', 'order_', 'family', 'genus', 'code']\n",
    "    sis_clean = pd.DataFrame(sis[columns])\n",
    "    sis_clean = sis_clean.astype(dtype={\"id_no\": \"uint32\",\n",
    "                                        'kingdom': \"category\",\n",
    "                                        'phylum': \"category\",\n",
    "                                         'class': \"category\",\n",
    "                                         'order_': \"category\",\n",
    "                                         'family': \"category\",\n",
    "                                         'genus': \"category\",\n",
    "                                         'code': \"category\"})\n",
    "        \n",
    "    sis_clean_index = sis_clean.set_index('id_no')\n",
    "    \n",
    "    # set up parallel engines\n",
    "    import ipyparallel as ipp\n",
    "    c = ipp.Client()\n",
    "    \n",
    "    dview = c[:]\n",
    "    dview.push(dict(sis_clean_index=sis_clean_index))\n",
    "\n",
    "    # read result in chunk\n",
    "    species = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=chunksize, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})\n",
    "    \n",
    "    # run\n",
    "    result = []\n",
    "    total_rows = 0\n",
    "    counter = 0\n",
    "    \n",
    "    df_chunks = []\n",
    "    \n",
    "    for df_chunk in species:\n",
    "        # debug\n",
    "#         if counter == 101:                  \n",
    "#             break\n",
    "        \n",
    "        if counter % 10 == 0:\n",
    "            print('Processed rows: {:,} '.format(counter*chunksize))\n",
    "        \n",
    "        if counter % 100 == 0:\n",
    "            elapsed = time.time() - start_time\n",
    "            print('Elapsed time: {}'.format(str(timedelta(seconds=elapsed))))\n",
    "#             print('Memory usage of the output: {:,.1f} KB \\n---'.format(result[0].memory_usage().sum()/1024))\n",
    "                    \n",
    "        \n",
    "        df_chunks.append(df_chunk)\n",
    "        counter += 1\n",
    "        \n",
    "        if len(df_chunks) == 8:\n",
    "            ar = dview.map_async(worker, df_chunks)\n",
    "            # clear list\n",
    "            df_chunks = []\n",
    "            # once all workers have finished, concatenate result\n",
    "            result = [concat_chunk_results(ar.get() + result)]\n",
    "        else:\n",
    "            continue\n",
    "            \n",
    "\n",
    "    # check if unprocessed chunks\n",
    "    if df_chunks:\n",
    "        ar = dview.map_async(worker, df_chunks)\n",
    "        result = concat_chunk_results(ar.get() + result)\n",
    "    else:\n",
    "        result = result[0]\n",
    "        \n",
    "    # final push\n",
    "    result.to_csv('output_parallel.csv')\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The main bottleneck appears to be I/O. Most of the I/O occur in one process while other engine processes sit idle. Not seeing a massive performance boom due that may benefit from the increase of CPU cores or memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processed rows: 0 \n",
      "Elapsed time: 0:00:03.057600\n",
      "Processed rows: 80,000,000 \n",
      "Processed rows: 160,000,000 \n",
      "Processed rows: 240,000,000 \n",
      "Processed rows: 320,000,000 \n",
      "Processed rows: 400,000,000 \n",
      "Processed rows: 480,000,000 \n",
      "Processed rows: 560,000,000 \n",
      "Processed rows: 640,000,000 \n",
      "Processed rows: 720,000,000 \n",
      "Processed rows: 800,000,000 \n",
      "Elapsed time: 0:06:09.169200\n",
      "Processed rows: 880,000,000 \n",
      "Processed rows: 960,000,000 \n",
      "Processed rows: 1,040,000,000 \n",
      "Processed rows: 1,120,000,000 \n",
      "Processed rows: 1,200,000,000 \n",
      "Processed rows: 1,280,000,000 \n",
      "Processed rows: 1,360,000,000 \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_NO</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WDPAID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090406</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090407</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090408</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090409</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090410</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090411</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090412</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090413</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090414</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090415</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090416</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090417</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090418</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090419</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090420</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090421</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090422</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090423</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090424</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090425</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090426</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090427</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090428</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090429</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090430</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090431</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090432</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090433</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090434</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5090435</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4982421 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID_NO\n",
       "WDPAID        \n",
       "61           1\n",
       "62           1\n",
       "63           1\n",
       "64           1\n",
       "65           1\n",
       "66           1\n",
       "67           1\n",
       "68           1\n",
       "69           1\n",
       "70           1\n",
       "71           1\n",
       "72           1\n",
       "73           1\n",
       "74           1\n",
       "75           1\n",
       "76           1\n",
       "77           1\n",
       "78           1\n",
       "79           1\n",
       "80           1\n",
       "81           1\n",
       "82           1\n",
       "83           1\n",
       "84           1\n",
       "85           1\n",
       "86           1\n",
       "87           1\n",
       "88           1\n",
       "89           1\n",
       "90           1\n",
       "...        ...\n",
       "5090406      4\n",
       "5090407      4\n",
       "5090408      4\n",
       "5090409      4\n",
       "5090410      4\n",
       "5090411      4\n",
       "5090412      4\n",
       "5090413      4\n",
       "5090414      4\n",
       "5090415      4\n",
       "5090416      4\n",
       "5090417      4\n",
       "5090418      4\n",
       "5090419      4\n",
       "5090420      4\n",
       "5090421      4\n",
       "5090422      4\n",
       "5090423      4\n",
       "5090424      4\n",
       "5090425      4\n",
       "5090426      4\n",
       "5090427      4\n",
       "5090428      4\n",
       "5090429      4\n",
       "5090430      4\n",
       "5090431      4\n",
       "5090432      4\n",
       "5090433      4\n",
       "5090434      4\n",
       "5090435      4\n",
       "\n",
       "[4982421 rows x 1 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test_main_parallel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variable               Type        Data/Info\n",
      "--------------------------------------------\n",
      "analyse_chunk_mk2      function    <function analyse_chunk_m<...>k2 at 0x000000000AE0CD90>\n",
      "c                      Client      <ipyparallel.client.clien<...>ct at 0x000000000807BE80>\n",
      "chunk_worker           function    <function chunk_worker at 0x000000000AFEBB70>\n",
      "concat_chunk_results   function    <function concat_chunk_re<...>ts at 0x000000000AFEBAE8>\n",
      "ipp                    module      <module 'ipyparallel' fro<...>pyparallel\\\\__init__.py'>\n",
      "np                     module      <module 'numpy' from 'E:\\<...>ges\\\\numpy\\\\__init__.py'>\n",
      "pd                     module      <module 'pandas' from 'E:<...>es\\\\pandas\\\\__init__.py'>\n",
      "test_main_parallel     function    <function test_main_paral<...>el at 0x00000000092F1950>\n",
      "worker                 function    <function worker at 0x0000000008047950>\n"
     ]
    }
   ],
   "source": [
    "whos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Distributed + parallel\n",
    "\n",
    "The idea is to have a combination of engines running on the local machine plus any machine that are visible in LAN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It appears significant overheads incur probably due to sending messages across the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_main_parallel_distribute():\n",
    "    import time\n",
    "    from datetime import timedelta\n",
    "    start_time = time.time()\n",
    "    chunksize = 8_000_000\n",
    "    \n",
    "    # read species attributes\n",
    "    sis = pd.read_csv(\"sis_2017.csv\")\n",
    "\n",
    "    columns=['id_no', 'kingdom', 'phylum', 'class', 'order_', 'family', 'genus', 'code']\n",
    "    sis_clean = pd.DataFrame(sis[columns])\n",
    "    sis_clean = sis_clean.astype(dtype={\"id_no\": \"uint32\",\n",
    "                                        'kingdom': \"category\",\n",
    "                                        'phylum': \"category\",\n",
    "                                         'class': \"category\",\n",
    "                                         'order_': \"category\",\n",
    "                                         'family': \"category\",\n",
    "                                         'genus': \"category\",\n",
    "                                         'code': \"category\"})\n",
    "        \n",
    "    sis_clean_index = sis_clean.set_index('id_no')\n",
    "    \n",
    "    # worker functions\n",
    "\n",
    "    \n",
    "    # set up parallel engines\n",
    "    import ipyparallel as ipp\n",
    "    c = ipp.Client()\n",
    "    n_chunks = len(c.ids)\n",
    "    v = c[:]\n",
    "\n",
    "    # read result in chunk\n",
    "    species = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=chunksize, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})\n",
    "    \n",
    "    # run\n",
    "    result = []\n",
    "    total_rows = 0\n",
    "    counter = 0\n",
    "    \n",
    "    \n",
    "    \n",
    "    df_chunks = []\n",
    "    \n",
    "    for df_chunk in species:\n",
    "        # add chunk to df_chunks until its\n",
    "        df_chunks.append(df_chunk)\n",
    "        counter += 1\n",
    "        \n",
    "        # if\n",
    "        if counter % n_chunks != 0:\n",
    "            continue\n",
    "            \n",
    "        # map async\n",
    "        amr = v.map(analyse_chunk_mk3, df_chunks)\n",
    "\n",
    "        pending = set(amr.msg_ids)\n",
    "        while pending:\n",
    "            try:\n",
    "                # check every 2 seconds to aggregate\n",
    "                c.wait(pending, 2)\n",
    "            except parallel.TimeoutError:\n",
    "                # ignore timeouterrors, since they only mean that at least one isn't done\n",
    "                pass\n",
    "\n",
    "            # finished is the set of msg_ids that are complete\n",
    "            finished = pending.difference(c.outstanding)\n",
    "\n",
    "            # debug, if more than 100 chunks been processed\n",
    "            counter += len(finished)\n",
    "            print('Processed rows: {:,} '.format(counter*chunksize))\n",
    "            elapsed = time.time() - start_time\n",
    "            print('Elapsed time: {}'.format(str(timedelta(seconds=elapsed))))\n",
    "\n",
    "            if counter > 100:\n",
    "                break\n",
    "\n",
    "            # update pending to exclude those that just finished\n",
    "            pending = pending.difference(finished)\n",
    "\n",
    "            # get results already available\n",
    "            df_chunks = [c.get_result(msg_id) for msg_id in finished]\n",
    "\n",
    "            result = [concat_chunk_results(df_chunks + result)]\n",
    "\n",
    "    result[0].to_csv('output_parallel_dis.csv')\n",
    "    return result[0]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29]\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import time, random\n",
    "\n",
    "import ipyparallel as ipp\n",
    "\n",
    "# create client & view\n",
    "rc = ipp.Client()\n",
    "print(rc.ids)\n",
    "v = rc.load_balanced_view()\n",
    "\n",
    "def sleep_here(count, t):\n",
    "    \"\"\"simple function that takes args, prints a short message, sleeps for a time, and returns the same args\"\"\"\n",
    "    import time\n",
    "    time.sleep(t*count)\n",
    "    return count,t\n",
    "\n",
    "amr = v.map(sleep_here, range(100), [ random.random() for i in range(100) ], chunksize=2)\n",
    "\n",
    "pending = set(amr.msg_ids)\n",
    "while pending:\n",
    "    try:\n",
    "        rc.wait(pending, 1e-3)\n",
    "    except parallel.TimeoutError:\n",
    "        # ignore timeouterrors, since they only mean that at least one isn't done\n",
    "        pass\n",
    "    # finished is the set of msg_ids that are complete\n",
    "    finished = pending.difference(rc.outstanding)\n",
    "    # update pending to exclude those that just finished\n",
    "    pending = pending.difference(finished)\n",
    "    for msg_id in finished:\n",
    "        # we know these are done, so don't worry about blocking\n",
    "        ar = rc.get_result(msg_id)\n",
    "        print(\"job id {} finished on engine {}\".format(msg_id, ar.engine_id))\n",
    "        print(\"with stdout:\")\n",
    "        print('    ' + ar.stdout.replace('\\n', '\\n    ').rstrip())\n",
    "        print(\"and results:\")\n",
    "        \n",
    "        # note that each job in a map always returns a list of length chunksize\n",
    "        # even if chunksize == 1\n",
    "        for (count,t) in ar.get():\n",
    "            print(\"  item {}: slept for {}\".format(count, t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import ipyparallel as ipp\n",
    "c = ipp.Client()\n",
    "v = c[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],\n",
       " <DirectView [0, 1, 2, 3,...]>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.ids, v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(c.ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "species = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=200000, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunk = species.get_chunk()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_NO</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WDPAID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>927009</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927010</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927011</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927012</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927013</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927014</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927015</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927016</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927033</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927034</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927035</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927036</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927037</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927038</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927039</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>927040</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930076</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930077</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930078</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930079</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930080</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930081</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930082</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930083</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930084</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930085</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930086</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930087</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930088</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>930089</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984675</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984676</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984677</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984678</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984679</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984680</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3984737</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987881</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987882</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987883</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987884</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987885</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987886</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987887</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3987888</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991089</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991091</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991093</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3991094</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3994294</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3994295</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4000711</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4016707</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4016725</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4016731</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026283</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026284</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026285</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026286</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4035825</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>161861 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID_NO\n",
       "WDPAID        \n",
       "927009       1\n",
       "927010       1\n",
       "927011       1\n",
       "927012       1\n",
       "927013       1\n",
       "927014       1\n",
       "927015       1\n",
       "927016       1\n",
       "927033       1\n",
       "927034       1\n",
       "927035       1\n",
       "927036       1\n",
       "927037       1\n",
       "927038       1\n",
       "927039       1\n",
       "927040       1\n",
       "930076       1\n",
       "930077       1\n",
       "930078       1\n",
       "930079       1\n",
       "930080       1\n",
       "930081       1\n",
       "930082       1\n",
       "930083       1\n",
       "930084       1\n",
       "930085       1\n",
       "930086       1\n",
       "930087       1\n",
       "930088       1\n",
       "930089       1\n",
       "...        ...\n",
       "3984675      1\n",
       "3984676      1\n",
       "3984677      1\n",
       "3984678      1\n",
       "3984679      1\n",
       "3984680      1\n",
       "3984737      1\n",
       "3987881      1\n",
       "3987882      1\n",
       "3987883      1\n",
       "3987884      1\n",
       "3987885      1\n",
       "3987886      1\n",
       "3987887      1\n",
       "3987888      1\n",
       "3991089      1\n",
       "3991091      1\n",
       "3991093      1\n",
       "3991094      1\n",
       "3994294      1\n",
       "3994295      1\n",
       "4000711      1\n",
       "4016707      1\n",
       "4016725      1\n",
       "4016731      1\n",
       "4026283      1\n",
       "4026284      1\n",
       "4026285      1\n",
       "4026286      1\n",
       "4035825      1\n",
       "\n",
       "[161861 rows x 1 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analyse_chunk_mk3(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "ar = v.map_async(analyse_chunk_mk3, [chunk])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ar.ready()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy(chunk):\n",
    "    pass\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing it out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "v = c[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(c.ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "species = pd.read_csv(\"test_result2017_reso10_memo.csv.gz\", chunksize=8000000, \n",
    "                        skipinitialspace=True,\n",
    "                        dtype={'ID_NO': np.uint32, 'WDPAID': np.uint32})\n",
    "\n",
    "\n",
    "\n",
    "chunks = []\n",
    "counter = 0\n",
    "for chunk in species:\n",
    "    chunks.append(chunk)\n",
    "    counter += 1\n",
    "    if counter == 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concat_chunk_results(df_chunks):\n",
    "    # list of chunks to concat, \n",
    "    # more effcient to have a couple of dfs, than appending each df and sum\n",
    "    df_chunk = pd.concat(df_chunks)\n",
    "    return df_chunk.groupby('WDPAID').sum()\n",
    "\n",
    "def analyse_chunk_mk3(df_chunk):\n",
    "#     df_chunk = df_chunk[df_chunk.ID_NO.isin(sis_clean_index.index)]\n",
    "    return df_chunk.groupby('WDPAID').count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multi_version_dv():\n",
    "    import ipyparallel as ipp\n",
    "    c = ipp.Client()\n",
    "    v = c[:]\n",
    "    \n",
    "    start = time.time()\n",
    "    \n",
    "    amr = v.map_async(analyse_chunk_mk3, chunks)\n",
    "    result = concat_chunk_results(amr.get())\n",
    "    \n",
    "    print(time.time() - start)\n",
    "    return result\n",
    "\n",
    "def multi_version_load():\n",
    "    import ipyparallel as ipp\n",
    "    c = ipp.Client()\n",
    "    v = c.load_balanced_view()\n",
    "    \n",
    "    start = time.time()\n",
    "    \n",
    "    amr = v.map_async(analyse_chunk_mk3, chunks)\n",
    "    result = concat_chunk_results(amr.get())\n",
    "    \n",
    "    print(time.time() - start)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def single_version():\n",
    "    start = time.time()\n",
    "    \n",
    "    result = concat_chunk_results(analyse_chunk_mk3(chunk) for chunk in chunks)\n",
    "    \n",
    "    print(time.time() - start)\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.612799882888794\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_NO</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WDPAID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>216938</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216939</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216940</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216941</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216942</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216943</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216944</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216945</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216946</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216947</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216948</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216949</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218972</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218973</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218974</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218975</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218976</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218977</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218978</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218979</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218980</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218981</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218982</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218983</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218984</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218985</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218986</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218987</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218988</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218989</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089023</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089024</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089025</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089026</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089896</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089897</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089899</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089900</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089901</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089902</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089903</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089904</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089905</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089906</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089907</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089908</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089909</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089910</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089911</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089912</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089913</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089914</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089915</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089916</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089917</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089918</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089919</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089920</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089921</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089922</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4246677 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID_NO\n",
       "WDPAID        \n",
       "216938       1\n",
       "216939       1\n",
       "216940       1\n",
       "216941       1\n",
       "216942       1\n",
       "216943       1\n",
       "216944       1\n",
       "216945       1\n",
       "216946       1\n",
       "216947       1\n",
       "216948       1\n",
       "216949       1\n",
       "218972       1\n",
       "218973       1\n",
       "218974       1\n",
       "218975       1\n",
       "218976       1\n",
       "218977       1\n",
       "218978       1\n",
       "218979       1\n",
       "218980       1\n",
       "218981       1\n",
       "218982       1\n",
       "218983       1\n",
       "218984       1\n",
       "218985       1\n",
       "218986       1\n",
       "218987       1\n",
       "218988       1\n",
       "218989       1\n",
       "...        ...\n",
       "5089023      1\n",
       "5089024      1\n",
       "5089025      1\n",
       "5089026      1\n",
       "5089896      1\n",
       "5089897      1\n",
       "5089899      1\n",
       "5089900      1\n",
       "5089901      1\n",
       "5089902      1\n",
       "5089903      1\n",
       "5089904      1\n",
       "5089905      1\n",
       "5089906      1\n",
       "5089907      1\n",
       "5089908      1\n",
       "5089909      1\n",
       "5089910      1\n",
       "5089911      1\n",
       "5089912      1\n",
       "5089913      1\n",
       "5089914      1\n",
       "5089915      1\n",
       "5089916      1\n",
       "5089917      1\n",
       "5089918      1\n",
       "5089919      1\n",
       "5089920      1\n",
       "5089921      1\n",
       "5089922      1\n",
       "\n",
       "[4246677 rows x 1 columns]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multi_version_dv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.924800157546997\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID_NO</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WDPAID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>216938</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216939</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216940</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216941</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216942</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216943</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216944</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216945</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216946</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216947</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216948</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216949</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218972</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218973</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218974</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218975</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218976</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218977</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218978</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218979</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218980</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218981</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218982</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218983</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218984</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218985</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218986</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218987</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218988</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218989</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089023</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089024</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089025</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089026</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089896</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089897</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089899</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089900</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089901</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089902</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089903</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089904</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089905</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089906</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089907</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089908</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089909</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089910</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089911</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089912</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089913</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089914</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089915</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089916</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089917</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089918</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089919</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089920</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089921</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5089922</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4246677 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         ID_NO\n",
       "WDPAID        \n",
       "216938       1\n",
       "216939       1\n",
       "216940       1\n",
       "216941       1\n",
       "216942       1\n",
       "216943       1\n",
       "216944       1\n",
       "216945       1\n",
       "216946       1\n",
       "216947       1\n",
       "216948       1\n",
       "216949       1\n",
       "218972       1\n",
       "218973       1\n",
       "218974       1\n",
       "218975       1\n",
       "218976       1\n",
       "218977       1\n",
       "218978       1\n",
       "218979       1\n",
       "218980       1\n",
       "218981       1\n",
       "218982       1\n",
       "218983       1\n",
       "218984       1\n",
       "218985       1\n",
       "218986       1\n",
       "218987       1\n",
       "218988       1\n",
       "218989       1\n",
       "...        ...\n",
       "5089023      1\n",
       "5089024      1\n",
       "5089025      1\n",
       "5089026      1\n",
       "5089896      1\n",
       "5089897      1\n",
       "5089899      1\n",
       "5089900      1\n",
       "5089901      1\n",
       "5089902      1\n",
       "5089903      1\n",
       "5089904      1\n",
       "5089905      1\n",
       "5089906      1\n",
       "5089907      1\n",
       "5089908      1\n",
       "5089909      1\n",
       "5089910      1\n",
       "5089911      1\n",
       "5089912      1\n",
       "5089913      1\n",
       "5089914      1\n",
       "5089915      1\n",
       "5089916      1\n",
       "5089917      1\n",
       "5089918      1\n",
       "5089919      1\n",
       "5089920      1\n",
       "5089921      1\n",
       "5089922      1\n",
       "\n",
       "[4246677 rows x 1 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multi_version_load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dummy(sleep_time):\n",
    "    import time\n",
    "    time.sleep(sleep_time/10.0)\n",
    "    \n",
    "    return sleep_time\n",
    "\n",
    "import ipyparallel as ipp\n",
    "c = ipp.Client()\n",
    "v = c[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "amr = v.map_async(dummy, [1,2,3,4,5,6,7,8,9,2,3,5,6,7,10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amr.ready()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 5, 6, 7, 10]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amr.get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['849c1a3a-c31532a26a147538bb325b30',\n",
       " '41d0f9da-fe60d23a9c414003ac7e7bb4',\n",
       " '9b5f4e89-bc19265d84f1591ba682b425',\n",
       " '9951792b-b580ee9eb126bdf3063a6a2c',\n",
       " '4489a66a-4142f438b8f8a347048427a1',\n",
       " 'd618a70d-9a3e636058f3bde40af4c757',\n",
       " 'b009145f-236f6555fe22e33daf2cd294',\n",
       " 'd9a5faab-46495f2101bd1ab283a41649',\n",
       " '417476ea-5c13b6bc3f9b1fb4e9c7e616',\n",
       " '06da5303-08de2687af26f6444c7b8838']"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amr.msg_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.get_result('849c1a3a-c31532a26a147538bb325b30').get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}