TanzaniaTopSoilData_LogRegModel.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TanzaniaTopSoilData_LogRegModel.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOqZ/XBNAO50AjYUrOtAkga",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kvinne-anc/Tanzania-Soil-Topography-Waterpumps-Project/blob/main/TanzaniaTopSoilData_LogRegModel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xfsArHKBKEa1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "034e7ee5-cb6c-40bf-aec1-30a2bc1e3221"
      },
      "source": [
        "import sys\n",
        "\n",
        "# If you're on Colab:\n",
        "if 'google.colab' in sys.modules:\n",
        "    DATA_PATH = 'https://raw.githubusercontent.com/LambdaSchool/DS-Unit-2-Kaggle-Challenge/master/data/'\n",
        "    !pip install category_encoders==2.*\n",
        "    !pip install pandas-profiling==2.*"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: category_encoders==2.* in /usr/local/lib/python3.6/dist-packages (2.2.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (1.18.5)\n",
            "Requirement already satisfied: pandas>=0.21.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (1.0.5)\n",
            "Requirement already satisfied: patsy>=0.5.1 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (0.5.1)\n",
            "Requirement already satisfied: statsmodels>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (0.10.2)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from category_encoders==2.*) (0.22.2.post1)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders==2.*) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.21.1->category_encoders==2.*) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from patsy>=0.5.1->category_encoders==2.*) (1.15.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20.0->category_encoders==2.*) (0.16.0)\n",
            "Requirement already satisfied: pandas-profiling==2.* in /usr/local/lib/python3.6/dist-packages (2.9.0)\n",
            "Requirement already satisfied: missingno>=0.4.2 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.4.2)\n",
            "Requirement already satisfied: requests>=2.23.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (2.23.0)\n",
            "Requirement already satisfied: visions[type_image_path]==0.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.5.0)\n",
            "Requirement already satisfied: scipy>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (1.4.1)\n",
            "Requirement already satisfied: phik>=0.9.10 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.10.0)\n",
            "Requirement already satisfied: tqdm>=4.43.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (4.49.0)\n",
            "Requirement already satisfied: ipywidgets>=7.5.1 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (7.5.1)\n",
            "Requirement already satisfied: matplotlib>=3.2.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (3.2.2)\n",
            "Requirement already satisfied: attrs>=19.3.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (20.2.0)\n",
            "Requirement already satisfied: confuse>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (1.3.0)\n",
            "Requirement already satisfied: jinja2>=2.11.1 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (2.11.2)\n",
            "Requirement already satisfied: pandas!=1.0.0,!=1.0.1,!=1.0.2,!=1.1.0,>=0.25.3 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (1.0.5)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.16.0)\n",
            "Requirement already satisfied: seaborn>=0.10.1 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.10.1)\n",
            "Requirement already satisfied: htmlmin>=0.1.12 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.1.12)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (1.18.5)\n",
            "Requirement already satisfied: tangled-up-in-unicode>=0.0.6 in /usr/local/lib/python3.6/dist-packages (from pandas-profiling==2.*) (0.0.6)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.23.0->pandas-profiling==2.*) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.23.0->pandas-profiling==2.*) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.23.0->pandas-profiling==2.*) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.23.0->pandas-profiling==2.*) (3.0.4)\n",
            "Requirement already satisfied: networkx>=2.4 in /usr/local/lib/python3.6/dist-packages (from visions[type_image_path]==0.5.0->pandas-profiling==2.*) (2.5)\n",
            "Requirement already satisfied: imagehash; extra == \"type_image_path\" in /usr/local/lib/python3.6/dist-packages (from visions[type_image_path]==0.5.0->pandas-profiling==2.*) (4.1.0)\n",
            "Requirement already satisfied: Pillow; extra == \"type_image_path\" in /usr/local/lib/python3.6/dist-packages (from visions[type_image_path]==0.5.0->pandas-profiling==2.*) (7.0.0)\n",
            "Requirement already satisfied: numba>=0.38.1 in /usr/local/lib/python3.6/dist-packages (from phik>=0.9.10->pandas-profiling==2.*) (0.48.0)\n",
            "Requirement already satisfied: ipython>=4.0.0; python_version >= \"3.3\" in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.5.1->pandas-profiling==2.*) (5.5.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.5.1->pandas-profiling==2.*) (5.0.7)\n",
            "Requirement already satisfied: traitlets>=4.3.1 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.5.1->pandas-profiling==2.*) (4.3.3)\n",
            "Requirement already satisfied: ipykernel>=4.5.1 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.5.1->pandas-profiling==2.*) (4.10.1)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.6/dist-packages (from ipywidgets>=7.5.1->pandas-profiling==2.*) (3.5.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.2.0->pandas-profiling==2.*) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.2.0->pandas-profiling==2.*) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.2.0->pandas-profiling==2.*) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.2.0->pandas-profiling==2.*) (2.8.1)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from confuse>=1.0.0->pandas-profiling==2.*) (3.13)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2>=2.11.1->pandas-profiling==2.*) (1.1.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas!=1.0.0,!=1.0.1,!=1.0.2,!=1.1.0,>=0.25.3->pandas-profiling==2.*) (2018.9)\n",
            "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx>=2.4->visions[type_image_path]==0.5.0->pandas-profiling==2.*) (4.4.2)\n",
            "Requirement already satisfied: PyWavelets in /usr/local/lib/python3.6/dist-packages (from imagehash; extra == \"type_image_path\"->visions[type_image_path]==0.5.0->pandas-profiling==2.*) (1.1.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from imagehash; extra == \"type_image_path\"->visions[type_image_path]==0.5.0->pandas-profiling==2.*) (1.15.0)\n",
            "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.1->phik>=0.9.10->pandas-profiling==2.*) (0.31.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.1->phik>=0.9.10->pandas-profiling==2.*) (50.3.0)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.8.1)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (2.6.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (1.0.18)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (4.8.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.7.5)\n",
            "Requirement already satisfied: jupyter-core in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (4.6.3)\n",
            "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (2.6.0)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from nbformat>=4.2.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.2.0)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel>=4.5.1->ipywidgets>=7.5.1->pandas-profiling==2.*) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.6/dist-packages (from ipykernel>=4.5.1->ipywidgets>=7.5.1->pandas-profiling==2.*) (5.3.5)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.6/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (5.3.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.2.5)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0; python_version >= \"3.3\"->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.6.0)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel>=4.5.1->ipywidgets>=7.5.1->pandas-profiling==2.*) (19.0.2)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (1.5.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (5.6.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.8.3)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.4.4)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.6.0)\n",
            "Requirement already satisfied: entrypoints>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.3)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (3.2.0)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (1.4.2)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.6/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.8.4)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (0.5.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.5.1->pandas-profiling==2.*) (20.4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ONUyQCTIU0U-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "4c03b876-e48f-4214-e65e-2304812f9265"
      },
      "source": [
        "%%time \n",
        "\n",
        "# Important library for many geopython libraries\n",
        "!apt install gdal-bin python-gdal python3-gdal \n",
        "# Install rtree - Geopandas requirment\n",
        "!apt install python3-rtree \n",
        "# Install Geopandas\n",
        "!pip install git+git://github.com/geopandas/geopandas.git\n",
        "# Install descartes - Geopandas requirment\n",
        "!pip install descartes \n",
        "# Install Folium for Geographic data visualization\n",
        "!pip install folium\n",
        "# Install plotlyExpress\n",
        "!pip install plotly_express"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "gdal-bin is already the newest version (2.2.3+dfsg-2).\n",
            "python-gdal is already the newest version (2.2.3+dfsg-2).\n",
            "python3-gdal is already the newest version (2.2.3+dfsg-2).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 11 not upgraded.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "python3-rtree is already the newest version (0.8.3+ds-1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 11 not upgraded.\n",
            "Collecting git+git://github.com/geopandas/geopandas.git\n",
            "  Cloning git://github.com/geopandas/geopandas.git to /tmp/pip-req-build-s1f5lqcm\n",
            "  Running command git clone -q git://github.com/geopandas/geopandas.git /tmp/pip-req-build-s1f5lqcm\n",
            "Requirement already satisfied (use --upgrade to upgrade): geopandas==0.8.0+42.g16f3ba8 from git+git://github.com/geopandas/geopandas.git in /usr/local/lib/python3.6/dist-packages\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.6/dist-packages (from geopandas==0.8.0+42.g16f3ba8) (1.0.5)\n",
            "Requirement already satisfied: shapely>=1.6 in /usr/local/lib/python3.6/dist-packages (from geopandas==0.8.0+42.g16f3ba8) (1.7.1)\n",
            "Requirement already satisfied: fiona>=1.8 in /usr/local/lib/python3.6/dist-packages (from geopandas==0.8.0+42.g16f3ba8) (1.8.17)\n",
            "Requirement already satisfied: pyproj>=2.2.0 in /usr/local/lib/python3.6/dist-packages (from geopandas==0.8.0+42.g16f3ba8) (2.6.1.post1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->geopandas==0.8.0+42.g16f3ba8) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->geopandas==0.8.0+42.g16f3ba8) (2.8.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.24.0->geopandas==0.8.0+42.g16f3ba8) (1.18.5)\n",
            "Requirement already satisfied: click<8,>=4.0 in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (7.1.2)\n",
            "Requirement already satisfied: attrs>=17 in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (20.2.0)\n",
            "Requirement already satisfied: click-plugins>=1.0 in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (1.1.1)\n",
            "Requirement already satisfied: munch in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (2.5.0)\n",
            "Requirement already satisfied: six>=1.7 in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (1.15.0)\n",
            "Requirement already satisfied: cligj>=0.5 in /usr/local/lib/python3.6/dist-packages (from fiona>=1.8->geopandas==0.8.0+42.g16f3ba8) (0.5.0)\n",
            "Building wheels for collected packages: geopandas\n",
            "  Building wheel for geopandas (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for geopandas: filename=geopandas-0.8.0+42.g16f3ba8-py2.py3-none-any.whl size=963769 sha256=546cab6d440ade31dd137c41d2eaec5d2e0860fa87b0841f055f2db6d3a51a0a\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-3etpznoc/wheels/91/24/71/376c9c67192694168352afcccc2d264248f7e2cc6192997186\n",
            "Successfully built geopandas\n",
            "Requirement already satisfied: descartes in /usr/local/lib/python3.6/dist-packages (1.1.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from descartes) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib->descartes) (1.18.5)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->descartes) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->descartes) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->descartes) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->descartes) (1.2.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib->descartes) (1.15.0)\n",
            "Requirement already satisfied: folium in /usr/local/lib/python3.6/dist-packages (0.8.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from folium) (1.15.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from folium) (2.23.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from folium) (1.18.5)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.6/dist-packages (from folium) (2.11.2)\n",
            "Requirement already satisfied: branca>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from folium) (0.4.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->folium) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->folium) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->folium) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->folium) (2.10)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from jinja2->folium) (1.1.1)\n",
            "Requirement already satisfied: plotly_express in /usr/local/lib/python3.6/dist-packages (0.4.1)\n",
            "Requirement already satisfied: statsmodels>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (0.10.2)\n",
            "Requirement already satisfied: pandas>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (1.0.5)\n",
            "Requirement already satisfied: plotly>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (4.4.1)\n",
            "Requirement already satisfied: patsy>=0.5 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (0.5.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (1.18.5)\n",
            "Requirement already satisfied: scipy>=0.18 in /usr/local/lib/python3.6/dist-packages (from plotly_express) (1.4.1)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.20.0->plotly_express) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.20.0->plotly_express) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from plotly>=4.1.0->plotly_express) (1.15.0)\n",
            "Requirement already satisfied: retrying>=1.3.3 in /usr/local/lib/python3.6/dist-packages (from plotly>=4.1.0->plotly_express) (1.3.3)\n",
            "CPU times: user 83.4 ms, sys: 114 ms, total: 197 ms\n",
            "Wall time: 21 s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GZRjPBuRKI1R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 538
        },
        "outputId": "c12fa1c2-442c-4f97-c003-87ca594e8108"
      },
      "source": [
        "import pandas as pd\n",
        "import geopandas as gdp \n",
        "from shapely.geometry import Point\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns \n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "\n",
        "train = pd.merge(pd.read_csv('/content/tza_waterpumps.csv'), \n",
        "                 pd.read_csv(DATA_PATH+'waterpumps/train_labels.csv'), on= 'id')\n",
        "test = pd.read_csv('/content/test_tza_waterpumps.csv')\n",
        "\n",
        "sample_submission = pd.read_csv(DATA_PATH+'waterpumps/sample_submission.csv')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ParserError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mParserError\u001b[0m                               Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-29-23c1baa0e612>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m train = pd.merge(pd.read_csv('/content/tza_waterpumps.csv'), \n\u001b[0m\u001b[1;32m     11\u001b[0m                  pd.read_csv(DATA_PATH+'waterpumps/train_labels.csv'), on= 'id')\n\u001b[1;32m     12\u001b[0m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/test_tza_waterpumps.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    674\u001b[0m         )\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    452\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    453\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m     \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m         \u001b[0mparser\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   1131\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m         \u001b[0mnrows\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_validate_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"nrows\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1133\u001b[0;31m         \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;31m# May alter columns / col_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, nrows)\u001b[0m\n\u001b[1;32m   2035\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnrows\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2036\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2037\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnrows\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2038\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2039\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_first_chunk\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.read\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_low_memory\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._read_rows\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.raise_parser_error\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;31mParserError\u001b[0m: Error tokenizing data. C error: Expected 46 fields in line 67, saw 687\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WPihAie2_MD-"
      },
      "source": [
        "# Split train into train & val\n",
        "train, val = train_test_split(train, train_size=0.80, test_size=0.20, \n",
        "                              stratify=train['status_group'], random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X1uFv1AHKI4v",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "96e69e03-a555-4cac-9e46-2e6d4f53c5c7"
      },
      "source": [
        "\n",
        "train.shape, val.shape, test.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((12383, 47), (3096, 47), (14118, 46))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3LhiFzDuKI8R",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        },
        "outputId": "1d269395-6dd6-4c60-f16c-663cc60eb78e"
      },
      "source": [
        "train['status_group'].value_counts(normalize=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "functional                 0.524452\n",
              "non functional             0.411461\n",
              "functional needs repair    0.064087\n",
              "Name: status_group, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t_0_9RCEbJpk",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "fee95e3e-49bf-4280-c66f-4e64b5f44161"
      },
      "source": [
        "cat_col = train.columns[train.dtypes=='object']\n",
        "\n",
        "(cat_col)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['date_recorded', 'funder', 'installer', 'wpt_name', 'basin',\n",
              "       'subvillage', 'region', 'lga', 'ward', 'public_meeting', 'recorded_by',\n",
              "       'scheme_management', 'scheme_name', 'permit', 'extraction_type',\n",
              "       'extraction_type_group', 'extraction_type_class', 'management',\n",
              "       'management_group', 'payment', 'payment_type', 'water_quality',\n",
              "       'quality_group', 'quantity', 'quantity_group', 'source', 'source_type',\n",
              "       'source_class', 'waterpoint_type', 'waterpoint_type_group', 'SOIL_TYPE',\n",
              "       'MAIN_TYPE', 'geometry', 'status_group'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ULwucqRGbLE3"
      },
      "source": [
        "train[cat_col] = train[cat_col].astype('category')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WKMFs9U0bLHd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218
        },
        "outputId": "1d34ae8a-bfdf-4a93-af9a-7f5e2fb05c95"
      },
      "source": [
        "train.loc[0]\n",
        "train['lga'].cat.codes"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2672      88\n",
              "49889    110\n",
              "52171     96\n",
              "45347    121\n",
              "53494     57\n",
              "        ... \n",
              "52516    123\n",
              "3278      11\n",
              "31958     16\n",
              "9083     103\n",
              "41732     12\n",
              "Length: 45989, dtype: int8"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kKtsQwKbLK1"
      },
      "source": [
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xmh3t58LcCfn"
      },
      "source": [
        "trained = train.drop(['geometry', 'installer', 'extraction_type', 'construction_year', 'quantity', 'wpt_name', 'num_private', 'basin', 'region', 'district_code', 'population', 'public_meeting', 'recorded_by', 'id', 'scheme_management', 'scheme_name', 'permit', 'extraction_type_group', 'extraction_type_class', 'management_group', 'management', 'payment', 'payment_type', 'quality_group', 'source_type', 'source_class', 'source', 'date_recorded', 'waterpoint_type', 'waterpoint_type_group', 'status_group'], axis=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x9E-H8z7cDKp",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "outputId": "522bb0b9-4825-4243-aba8-609e9b8d2657"
      },
      "source": [
        "trained.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index_left</th>\n",
              "      <th>amount_tsh</th>\n",
              "      <th>funder</th>\n",
              "      <th>gps_height</th>\n",
              "      <th>longitude</th>\n",
              "      <th>latitude</th>\n",
              "      <th>subvillage</th>\n",
              "      <th>region_code</th>\n",
              "      <th>lga</th>\n",
              "      <th>ward</th>\n",
              "      <th>water_quality</th>\n",
              "      <th>quantity_group</th>\n",
              "      <th>AREA</th>\n",
              "      <th>PERIMETER</th>\n",
              "      <th>SOIL_TYPE</th>\n",
              "      <th>MAIN_TYPE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2672</th>\n",
              "      <td>54861.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Oxfam</td>\n",
              "      <td>0.0</td>\n",
              "      <td>30.691264</td>\n",
              "      <td>-2.847368</td>\n",
              "      <td>Nyakuguma</td>\n",
              "      <td>18.0</td>\n",
              "      <td>Ngara</td>\n",
              "      <td>Bugarama</td>\n",
              "      <td>soft</td>\n",
              "      <td>enough</td>\n",
              "      <td>0.692150</td>\n",
              "      <td>6.306691</td>\n",
              "      <td>FRh16</td>\n",
              "      <td>Haplic Ferrasols</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49889</th>\n",
              "      <td>39233.0</td>\n",
              "      <td>8000.0</td>\n",
              "      <td>Mission</td>\n",
              "      <td>1466.0</td>\n",
              "      <td>35.297519</td>\n",
              "      <td>-9.760209</td>\n",
              "      <td>Ganga</td>\n",
              "      <td>10.0</td>\n",
              "      <td>Songea Rural</td>\n",
              "      <td>Wino</td>\n",
              "      <td>soft</td>\n",
              "      <td>enough</td>\n",
              "      <td>1.112332</td>\n",
              "      <td>11.209680</td>\n",
              "      <td>AChX4</td>\n",
              "      <td>Haplic Acrisols</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>52171</th>\n",
              "      <td>10858.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Watu Wa Ujerumani</td>\n",
              "      <td>1188.0</td>\n",
              "      <td>34.031656</td>\n",
              "      <td>-1.146791</td>\n",
              "      <td>Kabwana</td>\n",
              "      <td>20.0</td>\n",
              "      <td>Rorya</td>\n",
              "      <td>Mkoma</td>\n",
              "      <td>soft</td>\n",
              "      <td>enough</td>\n",
              "      <td>0.085020</td>\n",
              "      <td>1.658887</td>\n",
              "      <td>CMx3</td>\n",
              "      <td>Chromic Cambisol</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45347</th>\n",
              "      <td>39840.0</td>\n",
              "      <td>100.0</td>\n",
              "      <td>Dhv</td>\n",
              "      <td>379.0</td>\n",
              "      <td>36.861544</td>\n",
              "      <td>-9.022433</td>\n",
              "      <td>Mwanza</td>\n",
              "      <td>5.0</td>\n",
              "      <td>Ulanga</td>\n",
              "      <td>Mbuga</td>\n",
              "      <td>unknown</td>\n",
              "      <td>enough</td>\n",
              "      <td>2.569950</td>\n",
              "      <td>29.917410</td>\n",
              "      <td>CMo21</td>\n",
              "      <td>Ferralic Cambiso</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>53494</th>\n",
              "      <td>6268.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>Netherland</td>\n",
              "      <td>0.0</td>\n",
              "      <td>33.754734</td>\n",
              "      <td>-3.157654</td>\n",
              "      <td>Nyanhumbili</td>\n",
              "      <td>17.0</td>\n",
              "      <td>Maswa</td>\n",
              "      <td>Buchambi</td>\n",
              "      <td>soft</td>\n",
              "      <td>enough</td>\n",
              "      <td>0.070808</td>\n",
              "      <td>1.707260</td>\n",
              "      <td>PLe7</td>\n",
              "      <td>Eutric Planosols</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       index_left  amount_tsh  ... SOIL_TYPE         MAIN_TYPE\n",
              "2672      54861.0         0.0  ...     FRh16  Haplic Ferrasols\n",
              "49889     39233.0      8000.0  ...     AChX4   Haplic Acrisols\n",
              "52171     10858.0         0.0  ...      CMx3  Chromic Cambisol\n",
              "45347     39840.0       100.0  ...     CMo21  Ferralic Cambiso\n",
              "53494      6268.0         0.0  ...      PLe7  Eutric Planosols\n",
              "\n",
              "[5 rows x 16 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "63We9QvhcDNV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "efd1cbbe-09a4-412a-ceb2-f0f159ecfd6d"
      },
      "source": [
        "numeric_features = trained.select_dtypes(include='number').columns.tolist()\n",
        "\n",
        "# Get a series with the cardinality of the nonnumeric features\n",
        "cardinality = trained.select_dtypes(exclude='number').nunique()\n",
        "\n",
        "# Get a list of all categorical features with cardinality <= 10000\n",
        "categorical_features = cardinality[cardinality <= 1000000].index.tolist()\n",
        "\n",
        "# Combine the lists \n",
        "\n",
        "features = numeric_features + categorical_features\n",
        "print(features)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['index_left', 'amount_tsh', 'gps_height', 'longitude', 'latitude', 'region_code', 'AREA', 'PERIMETER', 'funder', 'subvillage', 'lga', 'ward', 'water_quality', 'quantity_group', 'SOIL_TYPE', 'MAIN_TYPE']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-2g0nu-RcDT-"
      },
      "source": [
        "target = train['status_group']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WxRCb6GdcDQv"
      },
      "source": [
        "X_train = trained[features]\n",
        "y_train = train['status_group']\n",
        "X_val = val[features]\n",
        "y_val = val['status_group']\n",
        "X_test = test[features]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZOD5MA6sdcAz"
      },
      "source": [
        "import category_encoders as ce\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import make_pipeline\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "pipeline = make_pipeline(\n",
        "    ce.OneHotEncoder(use_cat_names=True), \n",
        "    SimpleImputer(strategy='mean'), \n",
        "    StandardScaler(), \n",
        "    LogisticRegression(multi_class='auto', solver='lbfgs', n_jobs=-1)\n",
        ")\n",
        "\n",
        "# Fit on train\n",
        "pipeline.fit(X_train, y_train)\n",
        "\n",
        "# Score on val\n",
        "print('Validation Accuracy', pipeline.score(X_val, y_val))\n",
        "\n",
        "# Predict on test\n",
        "y_pred = pipeline.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t5T7usWUhU6q"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}