3_Missing-Data.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.2"},"colab":{"name":"3_Missing-Data.ipynb","provenance":[],"collapsed_sections":[]}},"cells":[{"cell_type":"markdown","metadata":{"id":"CCgvFltYAthG"},"source":["# Missing Data\n","\n","Let's show a few convenient methods to deal with Missing Data in pandas:"]},{"cell_type":"code","metadata":{"collapsed":true,"id":"eXiezGjnAthG"},"source":["import numpy as np\n","import pandas as pd"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"collapsed":true,"id":"qGyks6u-AthH"},"source":["df = pd.DataFrame({'A':[1,2,np.nan],\n","                  'B':[5,np.nan,np.nan],\n","                  'C':[1,2,3]})"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"3ft1Q2v9AthI","executionInfo":{"status":"ok","timestamp":1633966826132,"user_tz":-180,"elapsed":6,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"5965bc4f-e207-4034-994f-7f57f75fe1f5"},"source":["df"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>A</th>\n","      <th>B</th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1.0</td>\n","      <td>5.0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2.0</td>\n","      <td>NaN</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     A    B  C\n","0  1.0  5.0  1\n","1  2.0  NaN  2\n","2  NaN  NaN  3"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":81},"id":"T9blzWQ1AthI","executionInfo":{"status":"ok","timestamp":1633966839443,"user_tz":-180,"elapsed":327,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"3dd56bbf-422b-4b6b-de0c-ea257d2e669b"},"source":["df.dropna()"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>A</th>\n","      <th>B</th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1.0</td>\n","      <td>5.0</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     A    B  C\n","0  1.0  5.0  1"]},"metadata":{},"execution_count":4}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"FDLBjc2zAthJ","executionInfo":{"status":"ok","timestamp":1633966854802,"user_tz":-180,"elapsed":315,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"37d94c61-0393-424b-848f-f97cf7ffc1d3"},"source":["df.dropna(axis=1)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["   C\n","0  1\n","1  2\n","2  3"]},"metadata":{},"execution_count":5}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":112},"id":"zrvbNFqlAthJ","executionInfo":{"status":"ok","timestamp":1633966885419,"user_tz":-180,"elapsed":309,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"ab8a5d2c-a90f-479a-e1ac-f754d577335d"},"source":["df.dropna(thresh=2)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>A</th>\n","      <th>B</th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1.0</td>\n","      <td>5.0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2.0</td>\n","      <td>NaN</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     A    B  C\n","0  1.0  5.0  1\n","1  2.0  NaN  2"]},"metadata":{},"execution_count":8}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"HWu55WfMAthJ","executionInfo":{"status":"ok","timestamp":1633966898835,"user_tz":-180,"elapsed":312,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"8e455801-5e77-48b2-9feb-169c54ee319d"},"source":["df.fillna(value='FILL VALUE')"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>A</th>\n","      <th>B</th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1</td>\n","      <td>5</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2</td>\n","      <td>FILL VALUE</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>FILL VALUE</td>\n","      <td>FILL VALUE</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["            A           B  C\n","0           1           5  1\n","1           2  FILL VALUE  2\n","2  FILL VALUE  FILL VALUE  3"]},"metadata":{},"execution_count":9}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":143},"id":"FdeSsyhQBE34","executionInfo":{"status":"ok","timestamp":1633966916653,"user_tz":-180,"elapsed":300,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"7c1f4ebd-06f4-4a04-90fd-341d5684395f"},"source":["df.fillna(value=4)"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>A</th>\n","      <th>B</th>\n","      <th>C</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>1.0</td>\n","      <td>5.0</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>2.0</td>\n","      <td>4.0</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>4.0</td>\n","      <td>4.0</td>\n","      <td>3</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["     A    B  C\n","0  1.0  5.0  1\n","1  2.0  4.0  2\n","2  4.0  4.0  3"]},"metadata":{},"execution_count":10}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"MjGd29a1AthK","executionInfo":{"status":"ok","timestamp":1633966985514,"user_tz":-180,"elapsed":5,"user":{"displayName":"Engin Tunalı","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"04365200986728431923"}},"outputId":"aa5ad902-e0ed-4281-b47f-900fd61e2a93"},"source":["df['A'].fillna(value=df['A'].mean())"],"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0    1.0\n","1    2.0\n","2    1.5\n","Name: A, dtype: float64"]},"metadata":{},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"moaQxlCBBSjW"},"source":[""],"execution_count":null,"outputs":[]}]}