50_DataCleaning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM7rRfPBDTrnwc+yH7TwmmA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bahiensed/ml/blob/main/DataCleaning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from matplotlib import pyplot\n",
        "\n",
        "from numpy import arange\n",
        "from numpy import loadtxt\n",
        "from numpy import unique\n",
        "\n",
        "from pandas import read_csv\n",
        "\n",
        "from sklearn.feature_selection import VarianceThreshold"
      ],
      "metadata": {
        "id": "vZuUNhZFK4TW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "naGasEgjGw8G",
        "outputId": "da0f1983-468f-48ae-d704-78893f190c78"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0 238\n",
            "1 297\n",
            "2 927\n",
            "3 933\n",
            "4 179\n",
            "5 375\n",
            "6 820\n",
            "7 618\n",
            "8 561\n",
            "9 57\n",
            "10 577\n",
            "11 59\n",
            "12 73\n",
            "13 107\n",
            "14 53\n",
            "15 91\n",
            "16 893\n",
            "17 810\n",
            "18 170\n",
            "19 53\n",
            "20 68\n",
            "21 9\n",
            "22 1\n",
            "23 92\n",
            "24 9\n",
            "25 8\n",
            "26 9\n",
            "27 308\n",
            "28 447\n",
            "29 392\n",
            "30 107\n",
            "31 42\n",
            "32 4\n",
            "33 45\n",
            "34 141\n",
            "35 110\n",
            "36 3\n",
            "37 758\n",
            "38 9\n",
            "39 9\n",
            "40 388\n",
            "41 220\n",
            "42 644\n",
            "43 649\n",
            "44 499\n",
            "45 2\n",
            "46 937\n",
            "47 169\n",
            "48 286\n",
            "49 2\n"
          ]
        }
      ],
      "source": [
        "# summarize the number of unique values for each column using numpy\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = loadtxt(url, delimiter=',')\n",
        "\n",
        "# summarize the number of unique values in each column\n",
        "for i in range(data.shape[1]):\n",
        "  print(i, len(unique(data[:, i])))"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# summarize the number of unique values for each column using pandas\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "df = read_csv(url, header=None)\n",
        "\n",
        "# summarize the number of unique values in each column\n",
        "print(df.nunique())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mUs2i1NoJk_i",
        "outputId": "6e3208ac-0421-44e9-e46b-26bfd92abf9f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0     238\n",
            "1     297\n",
            "2     927\n",
            "3     933\n",
            "4     179\n",
            "5     375\n",
            "6     820\n",
            "7     618\n",
            "8     561\n",
            "9      57\n",
            "10    577\n",
            "11     59\n",
            "12     73\n",
            "13    107\n",
            "14     53\n",
            "15     91\n",
            "16    893\n",
            "17    810\n",
            "18    170\n",
            "19     53\n",
            "20     68\n",
            "21      9\n",
            "22      1\n",
            "23     92\n",
            "24      9\n",
            "25      8\n",
            "26      9\n",
            "27    308\n",
            "28    447\n",
            "29    392\n",
            "30    107\n",
            "31     42\n",
            "32      4\n",
            "33     45\n",
            "34    141\n",
            "35    110\n",
            "36      3\n",
            "37    758\n",
            "38      9\n",
            "39      9\n",
            "40    388\n",
            "41    220\n",
            "42    644\n",
            "43    649\n",
            "44    499\n",
            "45      2\n",
            "46    937\n",
            "47    169\n",
            "48    286\n",
            "49      2\n",
            "dtype: int64\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# delete columns with a single unique value\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = read_csv(url, header=None)\n",
        "\n",
        "# print data shape before drop\n",
        "print(data.shape)\n",
        "\n",
        "# get number of unique values for each column\n",
        "counts = data.nunique()\n",
        "\n",
        "# record columns to delete\n",
        "to_del = [i for i, v in enumerate(counts) if v == 1]\n",
        "print(to_del)\n",
        "\n",
        "# drop useless columns\n",
        "data.drop(to_del, axis=1, inplace=True)\n",
        "print(data.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l9oiB4xBKwM1",
        "outputId": "acb35af0-3c76-47bc-9962-783dff7547b5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(937, 50)\n",
            "[22]\n",
            "(937, 49)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = loadtxt(url, delimiter=',')\n",
        "\n",
        "# summarize the number of unique values in each column\n",
        "for i in range(data.shape[1]):\n",
        "  num = len(unique(data[:, i]))\n",
        "  percentage = float(num) / data.shape[0] * 100\n",
        "  if percentage < 1:\n",
        "    print('%d, %d, %.2f%%' % (i, num, percentage))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MVbvFKJ4L9fP",
        "outputId": "bb1dc3d3-f84c-44c2-8e0b-27236b9fb8d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "21, 9, 0.96%\n",
            "22, 1, 0.11%\n",
            "24, 9, 0.96%\n",
            "25, 8, 0.85%\n",
            "26, 9, 0.96%\n",
            "32, 4, 0.43%\n",
            "36, 3, 0.32%\n",
            "38, 9, 0.96%\n",
            "39, 9, 0.96%\n",
            "45, 2, 0.21%\n",
            "49, 2, 0.21%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# delete columns where number of unique values is less than 1% of the rows\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = read_csv(url, header=None)\n",
        "\n",
        "# print data shape before drop\n",
        "print(data.shape)\n",
        "\n",
        "# get number of unique values for each column\n",
        "counts = data.nunique()\n",
        "\n",
        "# record columns to delete\n",
        "to_del = [i for i, v in enumerate(counts) if (float(v)/df.shape[0]*100) < 1]\n",
        "print(to_del)\n",
        "\n",
        "# drop useless columns\n",
        "data.drop(to_del, axis=1, inplace=True)\n",
        "print(data.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KpoMnoN6AKVt",
        "outputId": "bb857ecf-9d10-48d9-e1e9-b2d0b5ffb0d8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(937, 50)\n",
            "[21, 22, 24, 25, 26, 32, 36, 38, 39, 45, 49]\n",
            "(937, 39)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# example of applying the variance threshold for feature selection\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = read_csv(url, header=None)\n",
        "\n",
        "# split data into inputs and outputs\n",
        "df = data.values\n",
        "X = df[:, :-1]\n",
        "y = df[:, -1]\n",
        "print(X.shape, y.shape)\n",
        "\n",
        "# define the transform\n",
        "transform = VarianceThreshold()\n",
        "\n",
        "# transform the input data\n",
        "X_sel = transform.fit_transform(X)\n",
        "print(X_sel.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wpRFRSAXIF4J",
        "outputId": "90701963-f8c0-4503-f8dd-9427e7d54972"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(937, 49) (937,)\n",
            "(937, 48)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# explore the effect of the variance thresholds on the number of selected features\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/bd49d2ce02fd180157dc7974bc51a190/raw/c0ae9e7efe95e144faa483492532a23680c38bbd/oil-spill.csv'\n",
        "data = read_csv(url, header=None)\n",
        "\n",
        "# split data into inputs and outputs\n",
        "df = data.values\n",
        "X = df[:, :-1]\n",
        "y = df[:, -1]\n",
        "print(X.shape, y.shape)\n",
        "\n",
        "# define thresholds to check\n",
        "thresholds = arange(.0, .55, .05)\n",
        "\n",
        "# apply transform with each threshold\n",
        "results = list()\n",
        "for t in thresholds:\n",
        "\n",
        "  # define the transform\n",
        "  transform = VarianceThreshold(threshold=t)\n",
        "\n",
        "  # transform the input data\n",
        "  X_sel = transform.fit_transform(X)\n",
        "\n",
        "  # determine the number of input features\n",
        "  n_features = X_sel.shape[1]\n",
        "  print('>Threshold=%.2f, Features=%d' % (t, n_features))\n",
        "\n",
        "  # store the result\n",
        "  results.append(n_features)\n",
        "\n",
        "# plot the threshold vs the number of selected features\n",
        "pyplot.plot(thresholds, results)\n",
        "pyplot.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 473
        },
        "id": "e8ORGrnTIuUg",
        "outputId": "9782d75e-2099-446d-8bc5-54bcf4de5301"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(937, 49) (937,)\n",
            ">Threshold=0.00, Features=48\n",
            ">Threshold=0.05, Features=37\n",
            ">Threshold=0.10, Features=36\n",
            ">Threshold=0.15, Features=35\n",
            ">Threshold=0.20, Features=35\n",
            ">Threshold=0.25, Features=35\n",
            ">Threshold=0.30, Features=35\n",
            ">Threshold=0.35, Features=35\n",
            ">Threshold=0.40, Features=35\n",
            ">Threshold=0.45, Features=33\n",
            ">Threshold=0.50, Features=31\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAd1UlEQVR4nO3deZxU5Z3v8c+vF7oBWbrpBpFuuloxggo2Ut0uaGKYMZrgAInARbPovVFjotF7nUky3kzG19XxZpm5GTWJMcg4MYnRICbKQKLXUQgSZSlkFUXZaVBo2WRfmt/8UadNB7vp6vVUnfq+X696UafOeap+z6v0y+E5p57H3B0REYmunLALEBGRzqWgFxGJOAW9iEjEKehFRCJOQS8iEnF5YRdwspKSEo/FYmGXISKSUZYsWfK+u5c2tS/tgj4Wi5FIJMIuQ0Qko5jZpub2aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYiLTNDvOXiUB//zHVZt3Rt2KSIiaSXtfjDVVrk5xoMvvc0Jd84f1CfsckRE0kZkzuh7FeYz9PTeLN64K+xSRETSSmSCHqCmspilm/dwrP5E2KWIiKSNlIPezHLNbKmZzQq2XzGzZcFjm5k920y7+kbHzeyowptSHSvm0LF63tj2QWd+jIhIRmnNGP2dwJtAbwB3v7xhh5k9AzzXTLtD7l7V5gpbobqyCIDFG3ZRVd63Kz5SRCTtpXRGb2ZlwFhgWhP7egNjgCbP6LtS/16FxPr1YJHG6UVEPpTq0M0DwDeBpga/JwAvuXtz4yWFZpYwswVmNqGpA8zsluCYRF1dXYolNa06Vkxi4y5OnPB2vY+ISFS0GPRmdg2ww92XNHPIdcCTp3iLCnePA9cDD5jZWScf4O5T3T3u7vHS0ibnzU9ZdWUxuw8eY13d/na9j4hIVKRyRj8aGGdmG4GngDFm9isAMysBaoDZzTV2963Bn+uBucDI9pV8ajWxYgAN34iIBFoMene/293L3D0GTAFedvcvBLsnArPc/XBTbc2syMwKguclJP/SWN0hlTejol8PSnsVsHiDgl5EBNp/H/0UThq2MbO4mTVctB0GJMxsOTAH+J67d2rQmxk1sWIWb9zdmR8jIpIxWjUFgrvPJTn80rB9RRPHJICbguevAsPbU2BbVMeKmL3yXbbuOcSgvt27+uNFRNJKpH4Z26C6MjlOr+EbEZGIBv3Q03vTqyBPF2RFRIho0OfmGBdWFOmMXkSEiAY9JCc4e2fHfnYfOBp2KSIioYps0FcH99MnNunuGxHJbpEN+hFlfeiWm6P56UUk60U26Avzc7mgvA+LNE4vIlkuskEPyeGbVVv3cvDo8bBLEREJTbSDvrKY4yecZZv3hF2KiEhoIh30oyqKMNMEZyKS3SId9L0L8xmmBcNFJMtFOugheT/965u0YLiIZK/IB70WDBeRbJcFQf/nBcNFRLJR5IO+f+9CKrRguIhkscgHPfx5wXB3LRguItknK4K+JqYFw0Uke2VF0DcsRLJogyY4E5HskxVBH+vXg5LTCnQ/vYhkpawIejOjprJIE5yJSFZKOejNLNfMlprZrGD752a2wcyWBY+qZtrdYGbvBI8bOqrw1qqOFbN1zyG27TkUVgkiIqFozRn9ncCbJ732DXevCh7LTm5gZsXAPcBFQA1wj5kVtbnadmhYiETDNyKSbVIKejMrA8YC01r5/lcBL7r7LnffDbwIXN3K9+gQwwYGC4Zr+EZEskyqZ/QPAN8ETp4w5n4zW2Fm/2pmBU20GwRsabRdG7z2F8zsFjNLmFmirq4uxZJa58MFw3VGLyJZpsWgN7NrgB3uvuSkXXcDQ4FqoBj4VluLcPep7h5393hpaWlb36ZFNZXFvL1dC4aLSHZJ5Yx+NDDOzDYCTwFjzOxX7v6uJx0B/p3kGPzJtgLljbbLgtdCEa9IXh7QguEikk1aDHp3v9vdy9w9BkwBXnb3L5jZQAAzM2ACsKqJ5i8AnzKzouAi7KeC10JxQXlfLRguIlknrx1tnzCzUsCAZcCtAGYWB25195vcfZeZ3QcsDtrc6+6hpWxhfi4jyrRguIhkl1YFvbvPBeYGz8c0c0wCuKnR9mPAY22usINVVxbz6Lz1HDpaT/duuWGXIyLS6bLil7GN1cSSC4Yv3aJxehHJDlkX9BcGC4Yv1gRnIpIlsi7o+3TPZ6gWDBeRLJJ1QQ9QEyvi9c27Oa4Fw0UkC2Rl0FdXFnPwqBYMF5HskJVBX6MJzkQki2Rl0H+4YLjupxeRLJCVQQ8QrygmsWm3FgwXkcjL2qCvqSxi14GjWjBcRCIva4O+YSESLRguIlGXtUFfWdKTktO66YKsiERe1ga9mVEdK1bQi0jkZW3QQ3L4pnb3Id7dqwXDRSS6sjroayobxul1Vi8i0ZXVQT9sYG9OK8jT8I2IRFpWB/2HC4brzhsRibCsDnpITnC2Zvs+9hzUguEiEk1ZH/QN99MnNuqsXkSiKeuDXguGi0jUZX3QF+bnMrysD4sU9CISUSkHvZnlmtlSM5sVbD9hZmvMbJWZPWZm+c20qzezZcFjZkcV3pGqY8WsrN3LoaP1YZciItLhWnNGfyfwZqPtJ4ChwHCgO3BTM+0OuXtV8BjXtjI7V01lkRYMF5HISinozawMGAtMa3jN3X/vAWARUNY5JXa+URXFWjBcRCIr1TP6B4BvAh9ZZDUYsvki8HwzbQvNLGFmC8xsQlMHmNktwTGJurq6FEvqOH2653POgF66ICsikdRi0JvZNcAOd1/SzCEPA/Pc/ZVm9le4exy4HnjAzM46+QB3n+rucXePl5aWplp7h6qpLNaC4SISSamc0Y8GxpnZRuApYIyZ/QrAzO4BSoG7mmvs7luDP9cDc4GR7Su5c1THkguGr35XC4aLSLS0GPTufre7l7l7DJgCvOzuXzCzm4CrgOvcvcnTYDMrMrOC4HkJyb80VndY9R1IE5yJSFS15z76R4ABwGvBrZP/CGBmcTNruGg7DEiY2XJgDvA9d0/LoB/Qu5DBxT00Ti8ikZPXmoPdfS7J4Rfcvcm27p4guNXS3V8leftlRqiOFTN3zQ7cHTMLuxwRkQ6R9b+MbaymsoidB46yru5A2KWIiHQYBX0jDROcafhGRKJEQd/IhwuG64KsiESIgr4RMyNeUawJzkQkUhT0J6mu1ILhIhItCvqT1MR0P72IRIuC/iTDBvaiZ7dcXZAVkchQ0J8kLzeHCyuKtLSgiESGgr4JNbFi1mzfx96Dx8IuRUSk3RT0TaiuLMYdEps0fCMimU9B34Sq8r7k55pusxSRSFDQN6EwP5cRZX31wykRiQQFfTOqY8Ws3LqXw8e0YLiIZDYFfTNqKos4Vu8s3bwn7FJERNpFQd+MUYODBcM1Ti8iGU5B34w+PbRguIhEg4L+FKpjxby+SQuGi0hmU9CfQnVlMQe0YLiIZDgF/SlogjMRiQIF/Smc3qeQ8uLuGqcXkYyWctCbWa6ZLTWzWcF2pZktNLO1ZvYbM+vWTLu7g2PWmNlVHVV4V6mOFZPYuBt3D7sUEZE2ac0Z/Z3Am422vw/8q7sPAXYDXz65gZmdC0wBzgOuBh42s9y2l9v1amLF7DxwlPXva8FwEclMKQW9mZUBY4FpwbYBY4AZwSGPAxOaaDoeeMrdj7j7BmAtUNPeortSdWWwYLjG6UUkQ6V6Rv8A8E2g4T7DfsAedz8ebNcCg5poNwjY0mi7yePM7BYzS5hZoq6uLsWSusaZwYLhmuBMRDJVi0FvZtcAO9x9SWcV4e5T3T3u7vHS0tLO+pg2aVgwXBdkRSRTpXJGPxoYZ2YbgadIDtk8CPQ1s7zgmDJgaxNttwLljbabOy6tVVcWs2XXId7bezjsUkREWq3FoHf3u929zN1jJC+svuzunwfmABODw24Anmui+UxgipkVmFklcDawqEMq70LVsSIADd+ISEZqz3303wLuMrO1JMfs/w3AzMaZ2b0A7v4GMB1YDTwP3ObuGTfv77kDeycXDNcFWRHJQHktH/Jn7j4XmBs8X08Td9C4+0ySZ/IN2/cD97enyLA1LBiucXoRyUT6ZWyKqrVguIhkKAV9iqpjWjBcRDKTgj5FIwdrwXARyUwK+hQV5ucyfFAfEht3h12KiEirKOhbobqymBW1e7RguIhkFAV9K9TEijlW7yzbogXDRSRzKOhbIV4RLBiu++lFJIMo6FuhYcFwXZAVkUyioG8lLRguIplGQd9K8VgRB47W8+a7+8IuRUQkJQr6VqoJFiLR8I2IZAoFfSsN7NOdsqLuuiArIhlDQd8GNbHkQiRaMFxEMoGCvg2qK7VguIhkDgV9G1THtGC4iGQOBX0bnFXak349tWC4iGQGBX0bmBnxWJEmOBORjKCgb6PqWDGbdx1k+wdaMFxE0puCvo0+vJ9e4/QikuYU9G304YLhGqcXkTTX4uLgZlYIzAMKguNnuPs9ZvYK0Cs4rD+wyN0nNNG+HlgZbG5293EdUnnIGhYM1xm9iKS7FoMeOAKMcff9ZpYPzDezP7j75Q0HmNkzwHPNtD/k7lUdUGvaiVcU88BLb7P30DH6dM8PuxwRkSa1OHTjSfuDzfzg8eFPQs2sNzAGeLZTKkxj1ZVFuMMSLRguImkspTF6M8s1s2XADuBFd1/YaPcE4CV3/6CZ5oVmljCzBWb2kaGd4P1vCY5J1NXVtaoDYRpZXpRcMHyDbrMUkfSVUtC7e30w/FIG1JjZ+Y12Xwc8eYrmFe4eB64HHjCzs5p4/6nuHnf3eGlpaSvKD1f3brmcP6iPLsiKSFpr1V037r4HmANcDWBmJUANMPsUbbYGf64H5gIj21hrWqqJacFwEUlvLQa9mZWaWd/geXfgSuCtYPdEYJa7N/mrITMrMrOC4HkJMBpY3RGFp4tqLRguImkulTP6gcAcM1sBLCY5Rj8r2DeFk4ZtzCxuZtOCzWFAwsyWk/yXwPfcPVJBH48VAZrgTETSV4u3V7r7CpoZbnH3K5p4LQHcFDx/FRjevhLTW98e3ThnQC/+/+rtTBg5iPLiHmGXJCLyF/TL2A4wpaacVdv2cvkP5nD9owv43dJaDh3VmL2IpAdLt1WS4vG4JxKJsMtota17DvHMklqeXrKFLbsO0asgj7+pOoPJ8XIuKOuDmYVdoohEmJktCe5w/Og+BX3HOnHCWbhhF08ntvD7Ve9y+NgJPjbgNCbHy5kwchAlpxWEXaKIRJCCPiQfHD7G7BXvMj2xhaWb95CXY4wZ2p/J8XKuOKeUvFyNnIlIx1DQp4F3tu/j6SW1/Pb1Wt7ff5TSXgV87sJBTBpVzpD+p4VdnohkOAV9GjlWf4K5a+qYntjCy2/toP6Ec+HgvkyOlzN2xEB6FWpyNBFpPQV9mtqx7zDPLt3K9EQta3fsp3t+Lp8ZPpDJ8TJqKot1AVdEUqagT3PuyV/WTk/U8h/Lt7H/yHEq+vVg0qgyrh1VxsA+3cMuUUTSnII+gxw6Ws/zb7zL9MW1vLZ+JzkGl59dyuR4OX99bn8K8nLDLlFE0pCCPkNt3nmQGUu2MGNJLdv2HqZvj3wmVA1iUryM887oE3Z5IpJGFPQZrv6E8+q695meqOWFN97j6PETnHdGbybHyxlfdQZ9e3QLu0QRCZmCPkL2HDzKzOXbeDpRy8qte+mWm8OV5w1gcrycy4aUkJujC7gi2UhBH1Grt33A9MQWnlu2ld0HjzGwTyETR5UxcVQZFf16hl2eiHQhBX3EHTlez0tv7uDpxBb++HYdJxwuqixmcrycTw8/nR7dUlkDXkQymYI+i7y39zDPvF7L04ktbNx5kNMK8rhmxEAmxcu5cHBf3ZsvElEK+izk7izeuJvpiS38fuW7HDxaz1mlPZkcL+ezFw6if6/CsEsUkQ6koM9y+48c5/fB5GqJTbvJzTE+eU5/JsXLGDO0P/maXE0k4yno5UPr6vYzY0ktzyypZce+I5Sc1o3PjhzEpHg5HxvQK+zyRKSNFPTyEcfrTzDvnTqmL67lpbe2c6zeqSrvy6R4GX9zwRn01uRqIhlFQS+ntHP/EZ5dto3pi7ewZvs+CvNz+PT5A5kUL+Piyn7k6N58kbTXrqA3s0JgHlBAcjHxGe5+j5n9HPgEsDc49EZ3X9ZE+xuAfwg2/8ndHz/V5ynow+PurNy6N7g3fxv7Dh+nvLg7Ey8s59pRgygr0sLnIumqvUFvQE93329m+cB84E7gVmCWu884RdtiIAHEAQeWAKPcfXdzbRT06eHwsXpeeOM9nk7UMn/t+5jBZUNKmBQv51PnDqAwX5OriaSTUwV9i7+k8eTfBPuDzfzgkep4z1XAi+6+KyjkReBq4MkU20tICvNzGV81iPFVg9iy62Bwb34tdzy5lN6FeYyvGkQ8VqT78qXTxPr1YERZ37DLiISUxujNLJfk2fgQ4Cfu/q1g6OYS4AjwEvD37n7kpHZ/BxS6+z8F298BDrn7v5x03C3ALQCDBw8etWnTpvb2SzrBiRPOgvU7mZ7Ywh9WvceR4yfCLkki7vZPDuF/XfkxzeGUgnad0QO4ez1QZWZ9gd+Z2fnA3cB7QDdgKvAt4N62FOjuU4P3IB6Pp9fVYflQTo5x6ZASLh1Swn2Hj7H9gyMtNxJpE+fReRv48Zy1vL55Nw9OGUlpr4Kwi8pYrZoExd33mNkc4OpGZ+VHzOzfgb9roslW4IpG22XA3DbUKWmmV2G+1reVTvX9iSMYFSviO8+uYuxDr/Cj60Zy0Zn9wi4rI7X4k0gzKw3O5DGz7sCVwFtmNjB4zYAJwKommr8AfMrMisysCPhU8JqISIsmx8t59rbR9CzI4/ppC3nkj+tIt1vCM0Eqv30fCMwxsxXAYpIXV2cBT5jZSmAlUAI0jMPHzWwaQHAR9r6g3WLg3oYLsyIiqRg2sDczbx/NVecN4Ht/eIubf7GEvQePhV1WRtEPpkQkI7g7P391I/fPfpOBfQt5+PpRDC/TkpoNTnUxVrNZiUhGMDP+++hKfvOVSzhe71z701d5YuEmDeWkQEEvIhllVEURs++4nIvP6se3f7eKu6Yv5+DR42GXldYU9CKScYp7duPnN1Zz15Uf49llWxn/4z+xdse+sMtKWwp6EclIOTnGHX91Nr/8Hxex68BRxv34Tzy3bGvYZaUlBb2IZLTLzi5h9h2Xc94ZvbnzqWV859lVHDleH3ZZaUVBLyIZ7/Q+hfz65ou55eNn8ssFm5j0yGts2XUw7LLShoJeRCIhPzeH//2ZYfzsi6PY8P4BrvnRfF56c3vYZaUFBb2IRMpV553OrK9fRllRd778eIIfPP8Wx+uzewI+Bb2IRE5Fv54889VLua5mMA/PXcfnpy1kx77DYZcVGgW9iERSYX4u3/3ccP7fpAtYXruHsQ/NZ8H6nWGXFQoFvYhE2rWjynjutsvoVZjH9Y8u4OG5azlxIrt+TaugF5HIO+f0Xsy8/TI+M3wgP3h+DTf/IsGeg0fDLqvLKOhFJCucVpDHj64byb3jz2PeO3WMfWg+K2r3hF1Wl1DQi0jWMDO+dEmMp2+9FICJP32NXy6I/sRoCnoRyTpV5X2Z9fXLGD2kH995dhV3PrWMA0eiOzGagl5EslJRz2782w3VfOOqc5i1Yhvjf/In3tkezYnRFPQikrVycozbPjmEX910EXsOJidGe3Zp9CZGU9CLSNa79KzkxGjDB/Xhf/5mGd/+3UoOH4vOxGgKehERYEDvQn5980V85RNn8sTCzUx85NXITIymoBcRCeTl5nD3p4fx6JfibN55kLEPvcKLqzN/YrQWg97MCs1skZktN7M3zOz/BK8/YWZrzGyVmT1mZvnNtK83s2XBY2ZHd0BEpKNdee4AZt9xOYP79eDmXyT47h/ezOiJ0VI5oz8CjHH3C4Aq4Gozuxh4AhgKDAe6Azc10/6Qu1cFj3EdUbSISGcrL+7BjFsv5fMXDeZnf1zP9dMWsuODzJwYrcWg96T9wWZ+8HB3/32wz4FFQFkn1iki0uUK83O5/7PDeeC/VbGydi+feWg+r657P+yyWi2lMXozyzWzZcAO4EV3X9hoXz7wReD5ZpoXmlnCzBaY2YRm3v+W4JhEXV1dK7sgItK5JowcxMzbR9Onex5fmLaQn8zJrInRUgp6d6939yqSZ+01ZnZ+o90PA/Pc/ZVmmle4exy4HnjAzM5q4v2nunvc3eOlpaWt7IKISOc7e0ByYrRrRpzBP7+whi8/vpjdBzJjYrRW3XXj7nuAOcDVAGZ2D1AK3HWKNluDP9cDc4GRbaxVRCRUPQvyeHBKFfdNOJ8/rd3JNT+az9LNu8Muq0Wp3HVTamZ9g+fdgSuBt8zsJuAq4Dp3b/JytJkVmVlB8LwEGA2s7qjiRUS6mpnxxYsrmPHVSwCY/LPXePzVjWk9MVoqZ/QDgTlmtgJYTHKMfhbwCDAAeC24dfIfAcwsbmbTgrbDgISZLSf5L4HvubuCXkQy3oiyvsy+4zI+fnYp98x8g9ufXMr+NJ0YzdLtb6F4PO6JRCLsMkREUnLihPPIvHX8ywtriJX05KefH8U5p/fq8jrMbElwPfQj9MtYEZF2yMkxvnbFEH5988XsO3yc8T+ZzzNLasMu6y8o6EVEOsDFZ/Zj9h2XUVXel799ejl3/3ZF2kyMpqAXEekg/XsV8qsvX8RtnzyLJxdt4XMPv8qmnQfCLktBLyLSkfJyc/jGVUN57MY4W/cc4pofzeeFN94LtSYFvYhIJxgzdACzvn4ZZ5b05Cu/XML9s1dzLKSJ0RT0IiKdpLy4B9NvvYQvXVLBo69s4LqpC3hvb9dPjKagFxHpRAV5udw7/nweum4kq9/9gLEPvcL8d7p2YjQFvYhIFxh3wRnMvH00xT278cXHFvLQS+902cRoCnoRkS4ypH8vnrt9NBOqBvHDF9/mxp8vZlcXTIymoBcR6UI9uuXxw8kX8H8/O5wF63Yy9qFXeL2TJ0ZT0IuIdDEz4/qLBvPbr11KXq4x+ZHXeGz+hk6bGE1BLyISkvMH9WHW7ZdzxTn9uXfWam7/9dJOGbfP6/B3FBGRlPXpkc+jXxrF1Hnr2Xf4ODk51uGfoaAXEQmZmfGVT3xk8b0Oo6EbEZGIU9CLiEScgl5EJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnHWWXMrtJWZ1QGb2vEWJUDXTvYcvmzrc7b1F9TnbNGePle4e2lTO9Iu6NvLzBLuHg+7jq6UbX3Otv6C+pwtOqvPGroREYk4Bb2ISMRFMeinhl1ACLKtz9nWX1Cfs0Wn9DlyY/QiIvKXonhGLyIijSjoRUQiLiOD3syuNrM1ZrbWzP6+if0FZvabYP9CM4t1fZUdK4U+f9zMXjez42Y2MYwaO1oKfb7LzFab2Qoze8nMKsKosyOl0OdbzWylmS0zs/lmdm4YdXaklvrc6LhrzczNLONvuUzhe77RzOqC73mZmd3Urg9094x6ALnAOuBMoBuwHDj3pGO+BjwSPJ8C/CbsurugzzFgBPALYGLYNXdRnz8J9AiefzVLvufejZ6PA54Pu+7O7nNwXC9gHrAAiIdddxd8zzcCP+6oz8zEM/oaYK27r3f3o8BTwPiTjhkPPB48nwH8lZl1/EKMXafFPrv7RndfAZwIo8BOkEqf57j7wWBzAVDWxTV2tFT6/EGjzZ5Apt9Nkcr/zwD3Ad8HDndlcZ0k1T53mEwM+kHAlkbbtcFrTR7j7seBvUC/Lqmuc6TS56hpbZ+/DPyhUyvqfCn12cxuM7N1wA+AO7qots7SYp/N7EKg3N1nd2VhnSjV/7avDYYlZ5hZeXs+MBODXuQvmNkXgDjwz2HX0hXc/SfufhbwLeAfwq6nM5lZDvBD4G/DrqWL/QcQc/cRwIv8eYSiTTIx6LcCjf92Kwtea/IYM8sD+gA7u6S6zpFKn6MmpT6b2V8D3wbGufuRLqqts7T2e34KmNCpFXW+lvrcCzgfmGtmG4GLgZkZfkG2xe/Z3Xc2+u95GjCqPR+YiUG/GDjbzCrNrBvJi60zTzpmJnBD8Hwi8LIHVzgyVCp9jpoW+2xmI4GfkQz5HSHU2NFS6fPZjTbHAu90YX2d4ZR9dve97l7i7jF3j5G8FjPO3RPhlNshUvmeBzbaHAe82a5PDPsKdBuvWn8GeJvkletvB6/dS/I/AIBC4GlgLbAIODPsmrugz9Ukx/oOkPzXyxth19wFff5PYDuwLHjMDLvmLujzg8AbQX/nAOeFXXNn9/mkY+eS4XfdpPg9fzf4npcH3/PQ9nyepkAQEYm4TBy6ERGRVlDQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQi7r8AN7tEsSUQvAQAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# locate rows of duplicate data\n",
        "# load the dataset\n",
        "url = 'https://gist.githubusercontent.com/dbahiense/370f4406813b4820a26f85c27c9b3f25/raw/2e2930cb8329356eff01696aacb79a1773aa9daa/iris.csv'\n",
        "data = read_csv(url, header=None)\n",
        "print(data.shape)\n",
        "\n",
        "# calculate duplicates\n",
        "dups = data.duplicated()\n",
        "\n",
        "# report if there are any duplicates\n",
        "print(dups.any())\n",
        "\n",
        "# list all duplicate rows\n",
        "print(data[dups])\n",
        "\n",
        "# delete rows of duplicate data from the dataset\n",
        "data.drop_duplicates(inplace=True)\n",
        "print(data.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4zZjjdAKLWvF",
        "outputId": "f51e75b5-4a4b-4877-bf0e-e501f74c9e2e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(150, 5)\n",
            "True\n",
            "       0    1    2    3               4\n",
            "34   4.9  3.1  1.5  0.1     Iris-setosa\n",
            "37   4.9  3.1  1.5  0.1     Iris-setosa\n",
            "142  5.8  2.7  5.1  1.9  Iris-virginica\n",
            "(147, 5)\n"
          ]
        }
      ]
    }
  ]
}