tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "15bd1013-a031-4415-ae50-946bbfc205bd",
   "metadata": {},
   "source": [
    "# Tutorial on dataframe, tabula, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "51b87b8b-75d3-4f3a-9296-f3a5ce317351",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tabula\n",
    "import pandas as pd\n",
    "from shared import (files, parse_int, headers_full)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aac2262f-5a82-426e-8a15-4d4b955e9113",
   "metadata": {},
   "source": [
    "## Load pdf into pages\n",
    "\n",
    "`read_pdf` loads pdf into a `list` of `pandas.core.frame.DataFrame`\n",
    "\n",
    "```python\n",
    "type(pages[0])\n",
    "pandas.core.frame.DataFrame\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b1cbbb43-ab1d-48c5-8859-36829fd7e673",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns_x = [100, 440, 560, 615, 675, 750, 825, 900, 976, 1051]\n",
    "area = [13, 0, 96, 97] #Y-top X-left Y-bottom X-Right\n",
    "pages = tabula.read_pdf(files[2021], pages='all', pandas_options={'header': None}, relative_area=True, area=area, columns=columns_x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f925b75-8c47-4d6c-8048-50543d3c42bf",
   "metadata": {},
   "source": [
    "## Concat everything into a `pandas.core.frame.DataFrame`\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "da6a5bed-9d93-4502-9d08-73444b0738d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "T = pd.concat(pages, ignore_index=True, sort=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "801ea8b8-b939-4f2c-a334-3aa99e306d61",
   "metadata": {},
   "source": [
    "## Rename the columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "003b14c7-0bd8-4f49-a333-48af8d9e9bad",
   "metadata": {},
   "outputs": [],
   "source": [
    "T.columns = headers_full\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7feee8ab-a452-40f1-857f-bb1e68e769d9",
   "metadata": {},
   "source": [
    "## How to access items?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4735a857-ffd3-4372-8cb0-eaa7903c227a",
   "metadata": {},
   "source": [
    "By column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "4a91d1c3-0a81-454d-a8d9-81536515fde7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0           1\n",
       "1           2\n",
       "2           5\n",
       "3           6\n",
       "4           8\n",
       "        ...  \n",
       "1126    1 452\n",
       "1127    1 453\n",
       "1128    1 454\n",
       "1129    1 455\n",
       "1130    1 456\n",
       "Name: nec, Length: 1131, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T[\"nec\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e8358d8-88a9-442f-8906-b704b7031fb8",
   "metadata": {},
   "source": [
    "single item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e8b932fe-9e5b-4401-8e14-c7b8c2a8ba36",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T[\"nec\"][5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bd52293-0b9c-474e-9172-a4c34ba700e4",
   "metadata": {},
   "source": [
    "or"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5d2c4d85-75b8-4411-b5bc-25fd79640f56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.loc[5,\"nec\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0c2146bc-1370-4878-b98a-22581d8677ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.set_option('display.max_rows', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "993e2d19-4147-4e46-8882-99da7d140d86",
   "metadata": {},
   "source": [
    "types"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5815a83c-3f63-442d-a87a-720171378ffa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nec                object\n",
       "name_raw           object\n",
       "t_scheduled        object\n",
       "t_done             object\n",
       "t_rate             object\n",
       "d_scheduled        object\n",
       "d_done             object\n",
       "d_rate             object\n",
       "total_scheduled    object\n",
       "total_done         object\n",
       "total_rate         object\n",
       "dtype: object"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "T.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1498135e-aa9b-45c7-a491-dca261144133",
   "metadata": {},
   "source": [
    "A column is a `pandas.core.series.Series`\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "11ad1690-827e-4b48-ba4d-648bb93d50ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.series.Series"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(T[\"nec\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}