1_Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(r\"C:\\Users\\Yasmin\\OneDrive\\Bachelor\\CS Project\\AppraiSet_complete_dataset.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>year_of_death</th>\n",
       "      <th>age</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>materials</th>\n",
       "      <th>length:</th>\n",
       "      <th>width</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>...</th>\n",
       "      <th>note</th>\n",
       "      <th>lower_estimate</th>\n",
       "      <th>upper_estimate</th>\n",
       "      <th>amount_sold</th>\n",
       "      <th>currency</th>\n",
       "      <th>sold_T_or_F</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_T_or_F_or_U</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925</td>\n",
       "      <td>NaN</td>\n",
       "      <td>96.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>oil on canvas, signed; signed on the stretcher</td>\n",
       "      <td>81</td>\n",
       "      <td>65</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5,265</td>\n",
       "      <td>...</td>\n",
       "      <td>To request a Condition Report, please contact;...</td>\n",
       "      <td>60000</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>352800.0</td>\n",
       "      <td>GBP</td>\n",
       "      <td>True</td>\n",
       "      <td>$74,930.94</td>\n",
       "      <td>$99,907.92</td>\n",
       "      <td>$440,593.93</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979</td>\n",
       "      <td>NaN</td>\n",
       "      <td>42.0</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>acrylic on canvas, signed, titled and dated 20...</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8,156</td>\n",
       "      <td>...</td>\n",
       "      <td>To request a Condition Report, please contact ...</td>\n",
       "      <td>50000</td>\n",
       "      <td>70000.0</td>\n",
       "      <td>201600.0</td>\n",
       "      <td>GBP</td>\n",
       "      <td>True</td>\n",
       "      <td>$62,442.45</td>\n",
       "      <td>$87,419.43</td>\n",
       "      <td>$251,767.96</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987</td>\n",
       "      <td>NaN</td>\n",
       "      <td>34.0</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>acrylic on gessoed board, signed and dated 201...</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,506</td>\n",
       "      <td>...</td>\n",
       "      <td>To request a Condition Report, please ; For an...</td>\n",
       "      <td>150000</td>\n",
       "      <td>200000.0</td>\n",
       "      <td>252000.0</td>\n",
       "      <td>GBP</td>\n",
       "      <td>True</td>\n",
       "      <td>$187,327.35</td>\n",
       "      <td>$249,769.80</td>\n",
       "      <td>$314,709.95</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>oil on linen</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37,290</td>\n",
       "      <td>...</td>\n",
       "      <td>To request a Condition Report, please ; For an...</td>\n",
       "      <td>60000</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>2253500.0</td>\n",
       "      <td>GBP</td>\n",
       "      <td>True</td>\n",
       "      <td>$74,930.94</td>\n",
       "      <td>$99,907.92</td>\n",
       "      <td>$2,814,281.22</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983</td>\n",
       "      <td>NaN</td>\n",
       "      <td>38.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>oil on canvas; signed, titled and dated 2018 o...</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,960</td>\n",
       "      <td>...</td>\n",
       "      <td>To request a Condition Report, please ; For an...</td>\n",
       "      <td>100000</td>\n",
       "      <td>150000.0</td>\n",
       "      <td>289800.0</td>\n",
       "      <td>GBP</td>\n",
       "      <td>True</td>\n",
       "      <td>$124,884.90</td>\n",
       "      <td>$187,327.35</td>\n",
       "      <td>$361,916.44</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        artist_name year_artist_born year_of_death   age  \\\n",
       "0        Etel Adnan             1925           NaN  96.0   \n",
       "1      Hilary Pecis             1979           NaN  42.0   \n",
       "2      Avery Singer             1987           NaN  34.0   \n",
       "3  Flora Yukhnovich             1990           NaN  31.0   \n",
       "4       Salman Toor             1983           NaN  38.0   \n",
       "\n",
       "                 title_of_work  \\\n",
       "0                     Untitled   \n",
       "1                 Summer Patio   \n",
       "2             Untitled (Study)   \n",
       "3  I'll Have What She's Having   \n",
       "4                 Garden Party   \n",
       "\n",
       "                                           materials length: width  height  \\\n",
       "0     oil on canvas, signed; signed on the stretcher      81    65     1.0   \n",
       "1  acrylic on canvas, signed, titled and dated 20...   106.9  76.3     1.0   \n",
       "2  acrylic on gessoed board, signed and dated 201...    76.2  98.5     1.0   \n",
       "3                                       oil on linen   169.5   220     1.0   \n",
       "4  oil on canvas; signed, titled and dated 2018 o...    99.5    80     1.0   \n",
       "\n",
       "     area  ...                                               note  \\\n",
       "0   5,265  ...  To request a Condition Report, please contact;...   \n",
       "1   8,156  ...  To request a Condition Report, please contact ...   \n",
       "2   7,506  ...  To request a Condition Report, please ; For an...   \n",
       "3  37,290  ...  To request a Condition Report, please ; For an...   \n",
       "4   7,960  ...  To request a Condition Report, please ; For an...   \n",
       "\n",
       "   lower_estimate upper_estimate  amount_sold currency sold_T_or_F  \\\n",
       "0           60000        80000.0     352800.0      GBP        True   \n",
       "1           50000        70000.0     201600.0      GBP        True   \n",
       "2          150000       200000.0     252000.0      GBP        True   \n",
       "3           60000        80000.0    2253500.0      GBP        True   \n",
       "4          100000       150000.0     289800.0      GBP        True   \n",
       "\n",
       "  lower_est_USD  upper_est_USD     sold_in_USD posthumous_T_or_F_or_U  \n",
       "0   $74,930.94     $99,907.92     $440,593.93                   FALSE  \n",
       "1   $62,442.45     $87,419.43     $251,767.96                   FALSE  \n",
       "2  $187,327.35    $249,769.80     $314,709.95                   FALSE  \n",
       "3   $74,930.94     $99,907.92   $2,814,281.22                   FALSE  \n",
       "4  $124,884.90    $187,327.35     $361,916.44                   FALSE  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10016, 33)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "artist_name                  2\n",
       "year_artist_born          2644\n",
       "year_of_death             5270\n",
       "age                       2662\n",
       "title_of_work                0\n",
       "materials                  974\n",
       "length:                      1\n",
       "width                       19\n",
       "height                      16\n",
       "area                        14\n",
       "measurement_2             6473\n",
       "length_2                  6475\n",
       "width_2                   6492\n",
       "height_2                  6508\n",
       "area_2                    6511\n",
       "measurement_3             9834\n",
       "length_3                  9834\n",
       "width_3                   9835\n",
       "height_3                  9841\n",
       "area_3                    9841\n",
       "unit_cm                     16\n",
       "year_of_work              4279\n",
       "age_at_work               4810\n",
       "note                         1\n",
       "lower_estimate               0\n",
       "upper_estimate               1\n",
       "amount_sold               2004\n",
       "currency                     0\n",
       "sold_T_or_F                  0\n",
       "lower_est_USD                0\n",
       "upper_est_USD                0\n",
       "sold_in_USD                  0\n",
       "posthumous_T_or_F_or_U       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are all of the NaNs we have\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>materials</th>\n",
       "      <th>length:</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>unit_cm</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_T_or_F_or_U</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925</td>\n",
       "      <td>96.0</td>\n",
       "      <td>oil on canvas, signed; signed on the stretcher</td>\n",
       "      <td>81</td>\n",
       "      <td>65</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5,265</td>\n",
       "      <td>cm</td>\n",
       "      <td>1973</td>\n",
       "      <td>48</td>\n",
       "      <td>$74,930.94</td>\n",
       "      <td>$99,907.92</td>\n",
       "      <td>$440,593.93</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979</td>\n",
       "      <td>42.0</td>\n",
       "      <td>acrylic on canvas, signed, titled and dated 20...</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8,156</td>\n",
       "      <td>cm</td>\n",
       "      <td>2019</td>\n",
       "      <td>40</td>\n",
       "      <td>$62,442.45</td>\n",
       "      <td>$87,419.43</td>\n",
       "      <td>$251,767.96</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987</td>\n",
       "      <td>34.0</td>\n",
       "      <td>acrylic on gessoed board, signed and dated 201...</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,506</td>\n",
       "      <td>cm</td>\n",
       "      <td>2016</td>\n",
       "      <td>29</td>\n",
       "      <td>$187,327.35</td>\n",
       "      <td>$249,769.80</td>\n",
       "      <td>$314,709.95</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990</td>\n",
       "      <td>31.0</td>\n",
       "      <td>oil on linen</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37,290</td>\n",
       "      <td>cm</td>\n",
       "      <td>2020</td>\n",
       "      <td>30</td>\n",
       "      <td>$74,930.94</td>\n",
       "      <td>$99,907.92</td>\n",
       "      <td>$2,814,281.22</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983</td>\n",
       "      <td>38.0</td>\n",
       "      <td>oil on canvas; signed, titled and dated 2018 o...</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,960</td>\n",
       "      <td>cm</td>\n",
       "      <td>2018</td>\n",
       "      <td>35</td>\n",
       "      <td>$124,884.90</td>\n",
       "      <td>$187,327.35</td>\n",
       "      <td>$361,916.44</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink and ink wash on paper</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4,938.50</td>\n",
       "      <td>cm</td>\n",
       "      <td>1960</td>\n",
       "      <td>36</td>\n",
       "      <td>$857.52</td>\n",
       "      <td>$1,286.28</td>\n",
       "      <td>$6,752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink and ink wash on paper</td>\n",
       "      <td>71</td>\n",
       "      <td>101</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,171.00</td>\n",
       "      <td>cm</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>$1,071.90</td>\n",
       "      <td>$2,143.80</td>\n",
       "      <td>$4,727.08</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink on paper laid down on canvas</td>\n",
       "      <td>73</td>\n",
       "      <td>102</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7,446.00</td>\n",
       "      <td>cm</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>$1,071.90</td>\n",
       "      <td>$2,143.80</td>\n",
       "      <td>$6,752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>i: mixed media on paperii,iii: ink on paper</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>813.75</td>\n",
       "      <td>cm</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>$1,607.85</td>\n",
       "      <td>$2,143.80</td>\n",
       "      <td>$5,672.49</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>i,ii,iii: ink on paper</td>\n",
       "      <td>30</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1,425.00</td>\n",
       "      <td>cm</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>$2,143.80</td>\n",
       "      <td>$3,215.70</td>\n",
       "      <td>$6,752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10016 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name year_artist_born   age  \\\n",
       "0            Etel Adnan             1925  96.0   \n",
       "1          Hilary Pecis             1979  42.0   \n",
       "2          Avery Singer             1987  34.0   \n",
       "3      Flora Yukhnovich             1990  31.0   \n",
       "4           Salman Toor             1983  38.0   \n",
       "...                 ...              ...   ...   \n",
       "10011         Fred Deux             1924  91.0   \n",
       "10012         Fred Deux             1924  91.0   \n",
       "10013         Fred Deux             1924  91.0   \n",
       "10014         Fred Deux             1924  91.0   \n",
       "10015         Fred Deux             1924  91.0   \n",
       "\n",
       "                                               materials length: width  \\\n",
       "0         oil on canvas, signed; signed on the stretcher      81    65   \n",
       "1      acrylic on canvas, signed, titled and dated 20...   106.9  76.3   \n",
       "2      acrylic on gessoed board, signed and dated 201...    76.2  98.5   \n",
       "3                                           oil on linen   169.5   220   \n",
       "4      oil on canvas; signed, titled and dated 2018 o...    99.5    80   \n",
       "...                                                  ...     ...   ...   \n",
       "10011                          ink and ink wash on paper    59.5    83   \n",
       "10012                          ink and ink wash on paper      71   101   \n",
       "10013                   ink on paper laid down on canvas      73   102   \n",
       "10014        i: mixed media on paperii,iii: ink on paper    46.5  17.5   \n",
       "10015                             i,ii,iii: ink on paper      30  47.5   \n",
       "\n",
       "                     title_of_work  height      area unit_cm year_of_work  \\\n",
       "0                         Untitled     1.0     5,265      cm         1973   \n",
       "1                     Summer Patio     1.0     8,156      cm         2019   \n",
       "2                 Untitled (Study)     1.0     7,506      cm         2016   \n",
       "3      I'll Have What She's Having     1.0    37,290      cm         2020   \n",
       "4                     Garden Party     1.0     7,960      cm         2018   \n",
       "...                            ...     ...       ...     ...          ...   \n",
       "10011              Détail d'infini     1.0  4,938.50      cm         1960   \n",
       "10012                   Sans titre     1.0  7,171.00      cm         1959   \n",
       "10013                     Untitled     1.0  7,446.00      cm         1959   \n",
       "10014           Untitled [3 works]     1.0    813.75      cm         1959   \n",
       "10015           Untitled [3 works]     1.0  1,425.00      cm         1959   \n",
       "\n",
       "      age_at_work lower_est_USD upper_est_USD     sold_in_USD  \\\n",
       "0              48   $74,930.94    $99,907.92     $440,593.93    \n",
       "1              40   $62,442.45    $87,419.43     $251,767.96    \n",
       "2              29  $187,327.35   $249,769.80     $314,709.95    \n",
       "3              30   $74,930.94    $99,907.92   $2,814,281.22    \n",
       "4              35  $124,884.90   $187,327.35     $361,916.44    \n",
       "...           ...           ...           ...             ...   \n",
       "10011          36      $857.52     $1,286.28       $6,752.97    \n",
       "10012          35    $1,071.90     $2,143.80       $4,727.08    \n",
       "10013          35    $1,071.90     $2,143.80       $6,752.97    \n",
       "10014          35    $1,607.85     $2,143.80       $5,672.49    \n",
       "10015          35    $2,143.80     $3,215.70       $6,752.97    \n",
       "\n",
       "      posthumous_T_or_F_or_U  \n",
       "0                      FALSE  \n",
       "1                      FALSE  \n",
       "2                      FALSE  \n",
       "3                      FALSE  \n",
       "4                      FALSE  \n",
       "...                      ...  \n",
       "10011                   TRUE  \n",
       "10012                   TRUE  \n",
       "10013                   TRUE  \n",
       "10014                   TRUE  \n",
       "10015                   TRUE  \n",
       "\n",
       "[10016 rows x 16 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#The selection of columns we think make most sense to keep, also regarding the amount of NaN values from above\n",
    "df = df[[ \"artist_name\", \"year_artist_born\", \"age\", \"materials\", \"length:\", \"width\", \"title_of_work\", \"height\", \"area\", \"unit_cm\", \"year_of_work\", \"age_at_work\", \"lower_est_USD\", \"upper_est_USD\",  \"sold_in_USD\", \"posthumous_T_or_F_or_U\" ]]\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#length: is the only column with a \":\", we rename and remove the \":\"\n",
    "df.rename(columns={'length:': 'length'}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "artist_name                  2\n",
       "year_artist_born          2644\n",
       "age                       2662\n",
       "materials                  974\n",
       "length                       1\n",
       "width                       19\n",
       "title_of_work                0\n",
       "height                      16\n",
       "area                        14\n",
       "unit_cm                     16\n",
       "year_of_work              4279\n",
       "age_at_work               4810\n",
       "lower_est_USD                0\n",
       "upper_est_USD                0\n",
       "sold_in_USD                  0\n",
       "posthumous_T_or_F_or_U       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5162, 16)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Even if we drop the NaNs we still have enough data\n",
    "df.dropna(inplace=True)\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            artist_name year_artist_born   age  \\\n",
      "0            Etel Adnan             1925  96.0   \n",
      "1          Hilary Pecis             1979  42.0   \n",
      "2          Avery Singer             1987  34.0   \n",
      "3      Flora Yukhnovich             1990  31.0   \n",
      "4           Salman Toor             1983  38.0   \n",
      "...                 ...              ...   ...   \n",
      "10011         Fred Deux             1924  91.0   \n",
      "10012         Fred Deux             1924  91.0   \n",
      "10013         Fred Deux             1924  91.0   \n",
      "10014         Fred Deux             1924  91.0   \n",
      "10015         Fred Deux             1924  91.0   \n",
      "\n",
      "                                               materials  length  width  \\\n",
      "0         oil on canvas, signed; signed on the stretcher    81.0   65.0   \n",
      "1      acrylic on canvas, signed, titled and dated 20...   106.9   76.3   \n",
      "2      acrylic on gessoed board, signed and dated 201...    76.2   98.5   \n",
      "3                                           oil on linen   169.5  220.0   \n",
      "4      oil on canvas; signed, titled and dated 2018 o...    99.5   80.0   \n",
      "...                                                  ...     ...    ...   \n",
      "10011                          ink and ink wash on paper    59.5   83.0   \n",
      "10012                          ink and ink wash on paper    71.0  101.0   \n",
      "10013                   ink on paper laid down on canvas    73.0  102.0   \n",
      "10014        i: mixed media on paperii,iii: ink on paper    46.5   17.5   \n",
      "10015                             i,ii,iii: ink on paper    30.0   47.5   \n",
      "\n",
      "                     title_of_work  height    area unit_cm year_of_work  \\\n",
      "0                         Untitled     1.0     NaN      cm         1973   \n",
      "1                     Summer Patio     1.0     NaN      cm         2019   \n",
      "2                 Untitled (Study)     1.0     NaN      cm         2016   \n",
      "3      I'll Have What She's Having     1.0     NaN      cm         2020   \n",
      "4                     Garden Party     1.0     NaN      cm         2018   \n",
      "...                            ...     ...     ...     ...          ...   \n",
      "10011              Détail d'infini     1.0     NaN      cm         1960   \n",
      "10012                   Sans titre     1.0     NaN      cm         1959   \n",
      "10013                     Untitled     1.0     NaN      cm         1959   \n",
      "10014           Untitled [3 works]     1.0  813.75      cm         1959   \n",
      "10015           Untitled [3 works]     1.0     NaN      cm         1959   \n",
      "\n",
      "      age_at_work lower_est_USD upper_est_USD     sold_in_USD  \\\n",
      "0              48   $74,930.94    $99,907.92     $440,593.93    \n",
      "1              40   $62,442.45    $87,419.43     $251,767.96    \n",
      "2              29  $187,327.35   $249,769.80     $314,709.95    \n",
      "3              30   $74,930.94    $99,907.92   $2,814,281.22    \n",
      "4              35  $124,884.90   $187,327.35     $361,916.44    \n",
      "...           ...           ...           ...             ...   \n",
      "10011          36      $857.52     $1,286.28       $6,752.97    \n",
      "10012          35    $1,071.90     $2,143.80       $4,727.08    \n",
      "10013          35    $1,071.90     $2,143.80       $6,752.97    \n",
      "10014          35    $1,607.85     $2,143.80       $5,672.49    \n",
      "10015          35    $2,143.80     $3,215.70       $6,752.97    \n",
      "\n",
      "      posthumous_T_or_F_or_U  \n",
      "0                      FALSE  \n",
      "1                      FALSE  \n",
      "2                      FALSE  \n",
      "3                      FALSE  \n",
      "4                      FALSE  \n",
      "...                      ...  \n",
      "10011                   TRUE  \n",
      "10012                   TRUE  \n",
      "10013                   TRUE  \n",
      "10014                   TRUE  \n",
      "10015                   TRUE  \n",
      "\n",
      "[5162 rows x 16 columns]\n"
     ]
    }
   ],
   "source": [
    "#The unit_cm column has values where mm turn up\n",
    "# We convert columns to numeric types\n",
    "numeric_columns = ['length', 'width', 'height', 'area']\n",
    "df[numeric_columns] = df[numeric_columns].apply(pd.to_numeric, errors='coerce')\n",
    "\n",
    "# We convert mm to cm for length, width, height, and area columns where unit is mm\n",
    "mm_indices = df.index[df['unit_cm'] == 'mm']  # To get indices where unit is mm\n",
    "\n",
    "# We update values in mm to cm (divide by 10)\n",
    "df.loc[mm_indices, ['length', 'width', 'height', 'area']] /= 10\n",
    "\n",
    "# We update the unit column to reflect the change to cm\n",
    "df.loc[mm_indices, 'unit_cm'] = 'cm'\n",
    "\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of 'mm' entries in unit_cm: 0\n"
     ]
    }
   ],
   "source": [
    "#We make sure we do not have any entries with mm left\n",
    "count_mms = (df[\"unit_cm\"] == \"mm\").sum()\n",
    "print(\"Number of 'mm' entries in unit_cm:\", count_mms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#All our values now have the same unit, no need to keep the unit_cm column \n",
    "df.drop(\"unit_cm\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>materials</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_T_or_F_or_U</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925</td>\n",
       "      <td>96.0</td>\n",
       "      <td>oil on canvas, signed; signed on the stretcher</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1973</td>\n",
       "      <td>48</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979</td>\n",
       "      <td>42.0</td>\n",
       "      <td>acrylic on canvas, signed, titled and dated 20...</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019</td>\n",
       "      <td>40</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987</td>\n",
       "      <td>34.0</td>\n",
       "      <td>acrylic on gessoed board, signed and dated 201...</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016</td>\n",
       "      <td>29</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990</td>\n",
       "      <td>31.0</td>\n",
       "      <td>oil on linen</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020</td>\n",
       "      <td>30</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983</td>\n",
       "      <td>38.0</td>\n",
       "      <td>oil on canvas; signed, titled and dated 2018 o...</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018</td>\n",
       "      <td>35</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>FALSE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink and ink wash on paper</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83.0</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1960</td>\n",
       "      <td>36</td>\n",
       "      <td>857.52</td>\n",
       "      <td>1286.28</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink and ink wash on paper</td>\n",
       "      <td>71.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>4727.08</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>ink on paper laid down on canvas</td>\n",
       "      <td>73.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>i: mixed media on paperii,iii: ink on paper</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>813.75</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>1607.85</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>5672.49</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924</td>\n",
       "      <td>91.0</td>\n",
       "      <td>i,ii,iii: ink on paper</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959</td>\n",
       "      <td>35</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>3215.70</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>TRUE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5162 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name year_artist_born   age  \\\n",
       "0            Etel Adnan             1925  96.0   \n",
       "1          Hilary Pecis             1979  42.0   \n",
       "2          Avery Singer             1987  34.0   \n",
       "3      Flora Yukhnovich             1990  31.0   \n",
       "4           Salman Toor             1983  38.0   \n",
       "...                 ...              ...   ...   \n",
       "10011         Fred Deux             1924  91.0   \n",
       "10012         Fred Deux             1924  91.0   \n",
       "10013         Fred Deux             1924  91.0   \n",
       "10014         Fred Deux             1924  91.0   \n",
       "10015         Fred Deux             1924  91.0   \n",
       "\n",
       "                                               materials  length  width  \\\n",
       "0         oil on canvas, signed; signed on the stretcher    81.0   65.0   \n",
       "1      acrylic on canvas, signed, titled and dated 20...   106.9   76.3   \n",
       "2      acrylic on gessoed board, signed and dated 201...    76.2   98.5   \n",
       "3                                           oil on linen   169.5  220.0   \n",
       "4      oil on canvas; signed, titled and dated 2018 o...    99.5   80.0   \n",
       "...                                                  ...     ...    ...   \n",
       "10011                          ink and ink wash on paper    59.5   83.0   \n",
       "10012                          ink and ink wash on paper    71.0  101.0   \n",
       "10013                   ink on paper laid down on canvas    73.0  102.0   \n",
       "10014        i: mixed media on paperii,iii: ink on paper    46.5   17.5   \n",
       "10015                             i,ii,iii: ink on paper    30.0   47.5   \n",
       "\n",
       "                     title_of_work  height    area year_of_work age_at_work  \\\n",
       "0                         Untitled     1.0     NaN         1973          48   \n",
       "1                     Summer Patio     1.0     NaN         2019          40   \n",
       "2                 Untitled (Study)     1.0     NaN         2016          29   \n",
       "3      I'll Have What She's Having     1.0     NaN         2020          30   \n",
       "4                     Garden Party     1.0     NaN         2018          35   \n",
       "...                            ...     ...     ...          ...         ...   \n",
       "10011              Détail d'infini     1.0     NaN         1960          36   \n",
       "10012                   Sans titre     1.0     NaN         1959          35   \n",
       "10013                     Untitled     1.0     NaN         1959          35   \n",
       "10014           Untitled [3 works]     1.0  813.75         1959          35   \n",
       "10015           Untitled [3 works]     1.0     NaN         1959          35   \n",
       "\n",
       "       lower_est_USD  upper_est_USD  sold_in_USD posthumous_T_or_F_or_U  \n",
       "0           74930.94       99907.92    440593.93                  FALSE  \n",
       "1           62442.45       87419.43    251767.96                  FALSE  \n",
       "2          187327.35      249769.80    314709.95                  FALSE  \n",
       "3           74930.94       99907.92   2814281.22                  FALSE  \n",
       "4          124884.90      187327.35    361916.44                  FALSE  \n",
       "...              ...            ...          ...                    ...  \n",
       "10011         857.52        1286.28      6752.97                   TRUE  \n",
       "10012        1071.90        2143.80      4727.08                   TRUE  \n",
       "10013        1071.90        2143.80      6752.97                   TRUE  \n",
       "10014        1607.85        2143.80      5672.49                   TRUE  \n",
       "10015        2143.80        3215.70      6752.97                   TRUE  \n",
       "\n",
       "[5162 rows x 15 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We need to remove the dollar sign and turn values into floats\n",
    "usd_remove = ['sold_in_USD', 'lower_est_USD', 'upper_est_USD']\n",
    "\n",
    "df[usd_remove] = df[usd_remove].replace('[\\$,]', '', regex=True).astype(float)\n",
    "\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# There are still some columns that are still objects, we convert these columns to float\n",
    "columns_to_float = ['year_of_work', 'age_at_work', 'year_artist_born',]  \n",
    "df[columns_to_float] = df[columns_to_float].astype(float)\n",
    "\n",
    "# We create dummy variables for the 'posthumous_T_or_F_or_U' column\n",
    "dummy_columns = pd.get_dummies(df['posthumous_T_or_F_or_U'], prefix='posthumous')\n",
    "\n",
    "# We concatenate the dummy columns with the original DataFrame\n",
    "df = pd.concat([df, dummy_columns], axis=1)\n",
    "\n",
    "\n",
    "# We drop the original 'posthumous_T_or_F_or_U' column\n",
    "df.drop('posthumous_T_or_F_or_U', axis=1, inplace=True)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We do not want boleans so we convert them to 0s and 1s, numericals\n",
    "bool_columns = ['posthumous_FALSE', 'posthumous_TRUE', 'posthumous_UNKNOWN']\n",
    "\n",
    "df[bool_columns] = df[bool_columns].astype(bool).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We drop rows where 'sold_in_USD' is 0\n",
    "df = df[df['sold_in_USD'] != 0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0              oil\n",
       "1          acrylic\n",
       "2          acrylic\n",
       "3              oil\n",
       "4              oil\n",
       "           ...    \n",
       "10011          ink\n",
       "10012          ink\n",
       "10013          ink\n",
       "10014           i:\n",
       "10015    i,ii,iii:\n",
       "Name: materials, Length: 4184, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We convert 'materials' column to string \n",
    "df['materials'] = df['materials'].astype(str)\n",
    "\n",
    "# We extract the first word from the 'materials' column using string manipulation methods as it suffices to get the materials used\n",
    "df['materials'] = df['materials'].str.split().str[0]\n",
    "\n",
    "# We take a look at the modified DataFrame\n",
    "df[\"materials\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#unwanted_words = ['a', 'green', 'black', 'the', 'unique', 'two', 'three', 'pale', 'early', 'six', 'new', 'multiple', 'baked', 'dayglo', 'injection', 'buttons', 'four', 'one', 'vintage', 'muralsized', 'matchsticks']\n",
    "\n",
    "#art_materials = [word for word in your_list if word not in unwanted_words]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We remove manually some input in the materials that are not real materials\n",
    "values_to_count = ['rhinestones', 'a', 'green', 'brushes', 'black', 'the', 'unique', 'two', 'three', 'fivecast', 'twopainted', 'pale', 'early', 'six', 'offsetlithographic', 'printed', 'new', 'multiple', 'baked', 'dayglo', 'injection', 'buttons', 'four', 'handmade', 'vintage', 'one', 'ferrotyped', 'muralsized', 'matchsticks']\n",
    "\n",
    "# Count occurrences of these values in 'materials' column\n",
    "counts = df['materials'].isin(values_to_count).sum()\n",
    "#print(counts) 26 such ocurrences\n",
    "\n",
    "# Filter and drop rows containing these values\n",
    "df = df[~df['materials'].isin(values_to_count)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We see what unique values we have for materials and categorize them into 10 categories\n",
    "\n",
    "# We extract unique materials from the 'materials' column\n",
    "unique_materials = df['materials'].str.split(';').explode().str.strip().str.split().str[0].unique()\n",
    "\n",
    "# We used ChatGPT to categorize the inputs/values from the materials column\n",
    "#1.\tPaints and Pigments: oil, acrylic, watercolor, gouache, tempera, ink, pigment\n",
    "#2.\tMetals and Sculpting Materials: bronze, stainless steel, aluminum, copper, wood, marble, resin, porcelain\n",
    "#3.\tPrintmaking and Graphic Arts: offset printing, lithography, screen printing, cibachrome, etching, charcoal\n",
    "#4.\tDrawing and Writing Tools: ballpoint, crayon, pencil, pen, markers\n",
    "#5.\tMixed Media and Miscellaneous: collage, fabric, leather, neon, vinyl, digital art, language, mixed\n",
    "#6.\tTraditional and Specialized Techniques: encaustic, silk screen, lacquer, terracotta, woodcut, stone carving\n",
    "#7.\tPhotography and Digital Art: photogram, inkjet, giclée print, digitally\n",
    "#8.\tSculpting and Carving: clay, fiberglass, metal, bamboo, wire, sand, glass, plaster\n",
    "#9.\tUnconventional and Unique Techniques: neon, LED, electronic, neon, language, multiples, found objects\n",
    "#10.\tOther Specific Techniques or Materials: Any material that doesn't fit neatly into the above categories.\n",
    "\n",
    "categories = {\n",
    "    1: ['oil', 'acrylic', 'spray', 'pastel', 'tempera', 'watercolor', 'watercolour', 'pigment', 'color'],\n",
    "    2: ['bronze', 'gold', 'patinated', 'wrought', 'alabaster', 'polished', 'copper', 'stainless', 'wood', 'cast', 'silver', 'steel', 'aluminum', 'walnut', 'plumbago', 'stone', 'corten', 'metallised', 'lead', 'burnished', 'brass', 'earthenware'],\n",
    "    3: ['cprint', 'offset', 'charcoal', 'screenprint', 'letterpress', 'etching'],\n",
    "    4: ['ballpoint', 'pen', 'signed', 'crayon', 'pencil', 'conte', 'marker', 'felttip'],\n",
    "    5: ['embroidery', 'synthetic', 'variable', 'mixed', 'ikb', 'paper', 'painted', 'polymer', 'glazed', 'alkyd', 'resin', 'polyuréthane', 'household', 'digital', 'book', 'fabric', 'gelatin', 'sump', 'vinyl', 'polyester', 'dye', 'offsetlithographin', 'monoprint', 'gypsum', 'printed', 'vegetable', 'archival', 'wool', 'digitally', 'enamel', 'monotype', 'cardboard', 'roll', 'woodcut', 'cement', 'lacquer', 'mirrored', 'oilstick', 'collage', 'pva', 'metal', 'bamboo', 'wires', 'wire', 'sand', 'epson', 'ektacolor', 'unframed', 'peacock', 'plastic', 'flashe', 'water', 'oxidation', 'porcelain', 'molave', 'handknitted', 'embossed', 'handembellished', 'polystone', 'silk', 'vacuumformed', 'cobblestones', 'feathers', 'diasecmounted', 'ukiyoe', 'giclee', 'giclée', 'handpainted', 'homemade', 'cotton', 'duratran', 'indian', 'varnished', 'handcut', 'fibreglass', 'vintage', 'humbrol'],\n",
    "    6: ['silkscreen', 'encaustic', 'terracotta', 'leather', 'screen', 'lambda', 'glass'],\n",
    "    7: ['ilfochrome', 'chromogenic', 'ink', 'cibachrome', 'photogram', 'inkjet', 'stencil', 'tar', 'photographic', 'epoxy', 'plaster'],\n",
    "    8: ['graphite', 'wax', 'chalk'],\n",
    "    9: ['language', 'led', 'electronic', 'neon', 'kinetic'],\n",
    "}\n",
    "import re\n",
    "\n",
    "# We define a function to clean materials list\n",
    "def clean_materials(materials_list):\n",
    "    cleaned_list = []\n",
    "    for material in materials_list:\n",
    "        if isinstance(material, str):\n",
    "            cleaned_material = re.sub(r'[^a-zA-Z\\s-]', '', material)\n",
    "            cleaned_material = ' '.join(cleaned_material.lower().split())\n",
    "            if cleaned_material:\n",
    "                cleaned_list.append(cleaned_material)\n",
    "    return cleaned_list\n",
    "\n",
    "\n",
    "# We clean the unique materials list\n",
    "cleaned_materials = clean_materials(unique_materials)\n",
    "\n",
    "# Function to categorize materials\n",
    "def categorize_material(material):\n",
    "    if isinstance(material, str):  # Check if material is a string\n",
    "        for category, materials in categories.items():\n",
    "            if material in materials:\n",
    "                return category\n",
    "    return 10  # Category for materials not fitting into defined categories\n",
    "\n",
    "\n",
    "# Apply cleaning and categorization to the 'materials' column\n",
    "df['Cleaned_Material'] = df['materials'].apply(lambda x: clean_materials(str(x).split(';')))\n",
    "df['Material_Category'] = df['Cleaned_Material'].apply(lambda x: categorize_material(x[0]) if x else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>materials</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_FALSE</th>\n",
       "      <th>posthumous_TRUE</th>\n",
       "      <th>posthumous_UNKNOWN</th>\n",
       "      <th>Cleaned_Material</th>\n",
       "      <th>Material_Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>oil</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>acrylic</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>acrylic</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>oil</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>oil</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        artist_name  year_artist_born   age materials  length  width  \\\n",
       "0        Etel Adnan            1925.0  96.0       oil    81.0   65.0   \n",
       "1      Hilary Pecis            1979.0  42.0   acrylic   106.9   76.3   \n",
       "2      Avery Singer            1987.0  34.0   acrylic    76.2   98.5   \n",
       "3  Flora Yukhnovich            1990.0  31.0       oil   169.5  220.0   \n",
       "4       Salman Toor            1983.0  38.0       oil    99.5   80.0   \n",
       "\n",
       "                 title_of_work  height  area  year_of_work  age_at_work  \\\n",
       "0                     Untitled     1.0   NaN        1973.0         48.0   \n",
       "1                 Summer Patio     1.0   NaN        2019.0         40.0   \n",
       "2             Untitled (Study)     1.0   NaN        2016.0         29.0   \n",
       "3  I'll Have What She's Having     1.0   NaN        2020.0         30.0   \n",
       "4                 Garden Party     1.0   NaN        2018.0         35.0   \n",
       "\n",
       "   lower_est_USD  upper_est_USD  sold_in_USD  posthumous_FALSE  \\\n",
       "0       74930.94       99907.92    440593.93                 1   \n",
       "1       62442.45       87419.43    251767.96                 1   \n",
       "2      187327.35      249769.80    314709.95                 1   \n",
       "3       74930.94       99907.92   2814281.22                 1   \n",
       "4      124884.90      187327.35    361916.44                 1   \n",
       "\n",
       "   posthumous_TRUE  posthumous_UNKNOWN Cleaned_Material  Material_Category  \n",
       "0                0                   0            [oil]                  1  \n",
       "1                0                   0        [acrylic]                  1  \n",
       "2                0                   0        [acrylic]                  1  \n",
       "3                0                   0            [oil]                  1  \n",
       "4                0                   0            [oil]                  1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_FALSE</th>\n",
       "      <th>posthumous_TRUE</th>\n",
       "      <th>posthumous_UNKNOWN</th>\n",
       "      <th>Cleaned_Material</th>\n",
       "      <th>Material_Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83.0</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>857.52</td>\n",
       "      <td>1286.28</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>4727.08</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>813.75</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1607.85</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>5672.49</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[i]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>3215.70</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[iiiiii]</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4158 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name  year_artist_born   age  length  width  \\\n",
       "0            Etel Adnan            1925.0  96.0    81.0   65.0   \n",
       "1          Hilary Pecis            1979.0  42.0   106.9   76.3   \n",
       "2          Avery Singer            1987.0  34.0    76.2   98.5   \n",
       "3      Flora Yukhnovich            1990.0  31.0   169.5  220.0   \n",
       "4           Salman Toor            1983.0  38.0    99.5   80.0   \n",
       "...                 ...               ...   ...     ...    ...   \n",
       "10011         Fred Deux            1924.0  91.0    59.5   83.0   \n",
       "10012         Fred Deux            1924.0  91.0    71.0  101.0   \n",
       "10013         Fred Deux            1924.0  91.0    73.0  102.0   \n",
       "10014         Fred Deux            1924.0  91.0    46.5   17.5   \n",
       "10015         Fred Deux            1924.0  91.0    30.0   47.5   \n",
       "\n",
       "                     title_of_work  height    area  year_of_work  age_at_work  \\\n",
       "0                         Untitled     1.0     NaN        1973.0         48.0   \n",
       "1                     Summer Patio     1.0     NaN        2019.0         40.0   \n",
       "2                 Untitled (Study)     1.0     NaN        2016.0         29.0   \n",
       "3      I'll Have What She's Having     1.0     NaN        2020.0         30.0   \n",
       "4                     Garden Party     1.0     NaN        2018.0         35.0   \n",
       "...                            ...     ...     ...           ...          ...   \n",
       "10011              Détail d'infini     1.0     NaN        1960.0         36.0   \n",
       "10012                   Sans titre     1.0     NaN        1959.0         35.0   \n",
       "10013                     Untitled     1.0     NaN        1959.0         35.0   \n",
       "10014           Untitled [3 works]     1.0  813.75        1959.0         35.0   \n",
       "10015           Untitled [3 works]     1.0     NaN        1959.0         35.0   \n",
       "\n",
       "       lower_est_USD  upper_est_USD  sold_in_USD  posthumous_FALSE  \\\n",
       "0           74930.94       99907.92    440593.93                 1   \n",
       "1           62442.45       87419.43    251767.96                 1   \n",
       "2          187327.35      249769.80    314709.95                 1   \n",
       "3           74930.94       99907.92   2814281.22                 1   \n",
       "4          124884.90      187327.35    361916.44                 1   \n",
       "...              ...            ...          ...               ...   \n",
       "10011         857.52        1286.28      6752.97                 0   \n",
       "10012        1071.90        2143.80      4727.08                 0   \n",
       "10013        1071.90        2143.80      6752.97                 0   \n",
       "10014        1607.85        2143.80      5672.49                 0   \n",
       "10015        2143.80        3215.70      6752.97                 0   \n",
       "\n",
       "       posthumous_TRUE  posthumous_UNKNOWN Cleaned_Material  Material_Category  \n",
       "0                    0                   0            [oil]                  1  \n",
       "1                    0                   0        [acrylic]                  1  \n",
       "2                    0                   0        [acrylic]                  1  \n",
       "3                    0                   0            [oil]                  1  \n",
       "4                    0                   0            [oil]                  1  \n",
       "...                ...                 ...              ...                ...  \n",
       "10011                1                   0            [ink]                  7  \n",
       "10012                1                   0            [ink]                  7  \n",
       "10013                1                   0            [ink]                  7  \n",
       "10014                1                   0              [i]                 10  \n",
       "10015                1                   0         [iiiiii]                 10  \n",
       "\n",
       "[4158 rows x 18 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We do not need the materials column anymore\n",
    "df.drop('materials', axis=1, inplace=True)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "artist_name              0\n",
       "year_artist_born         0\n",
       "age                      0\n",
       "length                   0\n",
       "width                    0\n",
       "title_of_work            0\n",
       "height                   0\n",
       "area                  2922\n",
       "year_of_work             0\n",
       "age_at_work              0\n",
       "lower_est_USD            0\n",
       "upper_est_USD            0\n",
       "sold_in_USD              0\n",
       "posthumous_FALSE         0\n",
       "posthumous_TRUE          0\n",
       "posthumous_UNKNOWN       0\n",
       "Cleaned_Material         0\n",
       "Material_Category        0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>area</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>posthumous_FALSE</th>\n",
       "      <th>posthumous_TRUE</th>\n",
       "      <th>posthumous_UNKNOWN</th>\n",
       "      <th>Cleaned_Material</th>\n",
       "      <th>Material_Category</th>\n",
       "      <th>act_area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5265.00</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>5265.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>8156.47</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "      <td>8156.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7505.70</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "      <td>7505.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>37290.00</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>37290.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7960.00</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>7960.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83.0</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4938.50</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>857.52</td>\n",
       "      <td>1286.28</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>4938.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7171.00</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>4727.08</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>7171.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7446.00</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>7446.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>813.75</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1607.85</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>5672.49</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[i]</td>\n",
       "      <td>10</td>\n",
       "      <td>813.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1425.00</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>3215.70</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>[iiiiii]</td>\n",
       "      <td>10</td>\n",
       "      <td>1425.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4158 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name  year_artist_born   age  length  width  \\\n",
       "0            Etel Adnan            1925.0  96.0    81.0   65.0   \n",
       "1          Hilary Pecis            1979.0  42.0   106.9   76.3   \n",
       "2          Avery Singer            1987.0  34.0    76.2   98.5   \n",
       "3      Flora Yukhnovich            1990.0  31.0   169.5  220.0   \n",
       "4           Salman Toor            1983.0  38.0    99.5   80.0   \n",
       "...                 ...               ...   ...     ...    ...   \n",
       "10011         Fred Deux            1924.0  91.0    59.5   83.0   \n",
       "10012         Fred Deux            1924.0  91.0    71.0  101.0   \n",
       "10013         Fred Deux            1924.0  91.0    73.0  102.0   \n",
       "10014         Fred Deux            1924.0  91.0    46.5   17.5   \n",
       "10015         Fred Deux            1924.0  91.0    30.0   47.5   \n",
       "\n",
       "                     title_of_work  height      area  year_of_work  \\\n",
       "0                         Untitled     1.0   5265.00        1973.0   \n",
       "1                     Summer Patio     1.0   8156.47        2019.0   \n",
       "2                 Untitled (Study)     1.0   7505.70        2016.0   \n",
       "3      I'll Have What She's Having     1.0  37290.00        2020.0   \n",
       "4                     Garden Party     1.0   7960.00        2018.0   \n",
       "...                            ...     ...       ...           ...   \n",
       "10011              Détail d'infini     1.0   4938.50        1960.0   \n",
       "10012                   Sans titre     1.0   7171.00        1959.0   \n",
       "10013                     Untitled     1.0   7446.00        1959.0   \n",
       "10014           Untitled [3 works]     1.0    813.75        1959.0   \n",
       "10015           Untitled [3 works]     1.0   1425.00        1959.0   \n",
       "\n",
       "       age_at_work  lower_est_USD  upper_est_USD  sold_in_USD  \\\n",
       "0             48.0       74930.94       99907.92    440593.93   \n",
       "1             40.0       62442.45       87419.43    251767.96   \n",
       "2             29.0      187327.35      249769.80    314709.95   \n",
       "3             30.0       74930.94       99907.92   2814281.22   \n",
       "4             35.0      124884.90      187327.35    361916.44   \n",
       "...            ...            ...            ...          ...   \n",
       "10011         36.0         857.52        1286.28      6752.97   \n",
       "10012         35.0        1071.90        2143.80      4727.08   \n",
       "10013         35.0        1071.90        2143.80      6752.97   \n",
       "10014         35.0        1607.85        2143.80      5672.49   \n",
       "10015         35.0        2143.80        3215.70      6752.97   \n",
       "\n",
       "       posthumous_FALSE  posthumous_TRUE  posthumous_UNKNOWN Cleaned_Material  \\\n",
       "0                     1                0                   0            [oil]   \n",
       "1                     1                0                   0        [acrylic]   \n",
       "2                     1                0                   0        [acrylic]   \n",
       "3                     1                0                   0            [oil]   \n",
       "4                     1                0                   0            [oil]   \n",
       "...                 ...              ...                 ...              ...   \n",
       "10011                 0                1                   0            [ink]   \n",
       "10012                 0                1                   0            [ink]   \n",
       "10013                 0                1                   0            [ink]   \n",
       "10014                 0                1                   0              [i]   \n",
       "10015                 0                1                   0         [iiiiii]   \n",
       "\n",
       "       Material_Category  act_area  \n",
       "0                      1   5265.00  \n",
       "1                      1   8156.47  \n",
       "2                      1   7505.70  \n",
       "3                      1  37290.00  \n",
       "4                      1   7960.00  \n",
       "...                  ...       ...  \n",
       "10011                  7   4938.50  \n",
       "10012                  7   7171.00  \n",
       "10013                  7   7446.00  \n",
       "10014                 10    813.75  \n",
       "10015                 10   1425.00  \n",
       "\n",
       "[4158 rows x 19 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We manually calculate the area column as there are a lot of NaNs in there and replace it with the act_area values we calculated\n",
    "df['act_area'] = df['height'] * df['width'] * df['length']\n",
    "df['area'] = df['act_area']\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop('area', axis=1, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>Cleaned_Material</th>\n",
       "      <th>Material_Category</th>\n",
       "      <th>act_area</th>\n",
       "      <th>posthumous_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>5265.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "      <td>8156.47</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>[acrylic]</td>\n",
       "      <td>1</td>\n",
       "      <td>7505.70</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>37290.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>[oil]</td>\n",
       "      <td>1</td>\n",
       "      <td>7960.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83.0</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>857.52</td>\n",
       "      <td>1286.28</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>4938.50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>4727.08</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>7171.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>[ink]</td>\n",
       "      <td>7</td>\n",
       "      <td>7446.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1607.85</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>5672.49</td>\n",
       "      <td>[i]</td>\n",
       "      <td>10</td>\n",
       "      <td>813.75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>3215.70</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>[iiiiii]</td>\n",
       "      <td>10</td>\n",
       "      <td>1425.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4158 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name  year_artist_born   age  length  width  \\\n",
       "0            Etel Adnan            1925.0  96.0    81.0   65.0   \n",
       "1          Hilary Pecis            1979.0  42.0   106.9   76.3   \n",
       "2          Avery Singer            1987.0  34.0    76.2   98.5   \n",
       "3      Flora Yukhnovich            1990.0  31.0   169.5  220.0   \n",
       "4           Salman Toor            1983.0  38.0    99.5   80.0   \n",
       "...                 ...               ...   ...     ...    ...   \n",
       "10011         Fred Deux            1924.0  91.0    59.5   83.0   \n",
       "10012         Fred Deux            1924.0  91.0    71.0  101.0   \n",
       "10013         Fred Deux            1924.0  91.0    73.0  102.0   \n",
       "10014         Fred Deux            1924.0  91.0    46.5   17.5   \n",
       "10015         Fred Deux            1924.0  91.0    30.0   47.5   \n",
       "\n",
       "                     title_of_work  height  year_of_work  age_at_work  \\\n",
       "0                         Untitled     1.0        1973.0         48.0   \n",
       "1                     Summer Patio     1.0        2019.0         40.0   \n",
       "2                 Untitled (Study)     1.0        2016.0         29.0   \n",
       "3      I'll Have What She's Having     1.0        2020.0         30.0   \n",
       "4                     Garden Party     1.0        2018.0         35.0   \n",
       "...                            ...     ...           ...          ...   \n",
       "10011              Détail d'infini     1.0        1960.0         36.0   \n",
       "10012                   Sans titre     1.0        1959.0         35.0   \n",
       "10013                     Untitled     1.0        1959.0         35.0   \n",
       "10014           Untitled [3 works]     1.0        1959.0         35.0   \n",
       "10015           Untitled [3 works]     1.0        1959.0         35.0   \n",
       "\n",
       "       lower_est_USD  upper_est_USD  sold_in_USD Cleaned_Material  \\\n",
       "0           74930.94       99907.92    440593.93            [oil]   \n",
       "1           62442.45       87419.43    251767.96        [acrylic]   \n",
       "2          187327.35      249769.80    314709.95        [acrylic]   \n",
       "3           74930.94       99907.92   2814281.22            [oil]   \n",
       "4          124884.90      187327.35    361916.44            [oil]   \n",
       "...              ...            ...          ...              ...   \n",
       "10011         857.52        1286.28      6752.97            [ink]   \n",
       "10012        1071.90        2143.80      4727.08            [ink]   \n",
       "10013        1071.90        2143.80      6752.97            [ink]   \n",
       "10014        1607.85        2143.80      5672.49              [i]   \n",
       "10015        2143.80        3215.70      6752.97         [iiiiii]   \n",
       "\n",
       "       Material_Category  act_area  posthumous_combined  \n",
       "0                      1   5265.00                    0  \n",
       "1                      1   8156.47                    0  \n",
       "2                      1   7505.70                    0  \n",
       "3                      1  37290.00                    0  \n",
       "4                      1   7960.00                    0  \n",
       "...                  ...       ...                  ...  \n",
       "10011                  7   4938.50                    1  \n",
       "10012                  7   7171.00                    1  \n",
       "10013                  7   7446.00                    1  \n",
       "10014                 10    813.75                    1  \n",
       "10015                 10   1425.00                    1  \n",
       "\n",
       "[4158 rows x 16 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#we want to have one combined column with posthumous unknown mapped to 0, True mapped to 1 and False mapped to 2\n",
    "# Create a new column and assign default value of 3 (Unknown)\n",
    "df['posthumous_combined'] = 3\n",
    "\n",
    "# Map the values from the dummy columns to the new combined column\n",
    "df['posthumous_combined'] = df['posthumous_TRUE'] * 1 + df['posthumous_FALSE'] * 0 + df['posthumous_UNKNOWN'] * 3\n",
    "\n",
    "\n",
    "# List of columns to drop\n",
    "columns_to_drop = ['posthumous_TRUE', 'posthumous_FALSE', 'posthumous_UNKNOWN']\n",
    "\n",
    "# Drop the columns\n",
    "df.drop(columns=columns_to_drop, inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>artist_name</th>\n",
       "      <th>year_artist_born</th>\n",
       "      <th>age</th>\n",
       "      <th>length</th>\n",
       "      <th>width</th>\n",
       "      <th>title_of_work</th>\n",
       "      <th>height</th>\n",
       "      <th>year_of_work</th>\n",
       "      <th>age_at_work</th>\n",
       "      <th>lower_est_USD</th>\n",
       "      <th>upper_est_USD</th>\n",
       "      <th>sold_in_USD</th>\n",
       "      <th>Material_Category</th>\n",
       "      <th>act_area</th>\n",
       "      <th>posthumous_combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Etel Adnan</td>\n",
       "      <td>1925.0</td>\n",
       "      <td>96.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1973.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>440593.93</td>\n",
       "      <td>1</td>\n",
       "      <td>5265.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hilary Pecis</td>\n",
       "      <td>1979.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>106.9</td>\n",
       "      <td>76.3</td>\n",
       "      <td>Summer Patio</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>62442.45</td>\n",
       "      <td>87419.43</td>\n",
       "      <td>251767.96</td>\n",
       "      <td>1</td>\n",
       "      <td>8156.47</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Avery Singer</td>\n",
       "      <td>1987.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>76.2</td>\n",
       "      <td>98.5</td>\n",
       "      <td>Untitled (Study)</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>249769.80</td>\n",
       "      <td>314709.95</td>\n",
       "      <td>1</td>\n",
       "      <td>7505.70</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Flora Yukhnovich</td>\n",
       "      <td>1990.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>169.5</td>\n",
       "      <td>220.0</td>\n",
       "      <td>I'll Have What She's Having</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>74930.94</td>\n",
       "      <td>99907.92</td>\n",
       "      <td>2814281.22</td>\n",
       "      <td>1</td>\n",
       "      <td>37290.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Salman Toor</td>\n",
       "      <td>1983.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>99.5</td>\n",
       "      <td>80.0</td>\n",
       "      <td>Garden Party</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>124884.90</td>\n",
       "      <td>187327.35</td>\n",
       "      <td>361916.44</td>\n",
       "      <td>1</td>\n",
       "      <td>7960.00</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10011</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>59.5</td>\n",
       "      <td>83.0</td>\n",
       "      <td>Détail d'infini</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1960.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>857.52</td>\n",
       "      <td>1286.28</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>7</td>\n",
       "      <td>4938.50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10012</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>Sans titre</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>4727.08</td>\n",
       "      <td>7</td>\n",
       "      <td>7171.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10013</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>Untitled</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1071.90</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>7</td>\n",
       "      <td>7446.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10014</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>46.5</td>\n",
       "      <td>17.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1607.85</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>5672.49</td>\n",
       "      <td>10</td>\n",
       "      <td>813.75</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10015</th>\n",
       "      <td>Fred Deux</td>\n",
       "      <td>1924.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>47.5</td>\n",
       "      <td>Untitled [3 works]</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1959.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>2143.80</td>\n",
       "      <td>3215.70</td>\n",
       "      <td>6752.97</td>\n",
       "      <td>10</td>\n",
       "      <td>1425.00</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4158 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            artist_name  year_artist_born   age  length  width  \\\n",
       "0            Etel Adnan            1925.0  96.0    81.0   65.0   \n",
       "1          Hilary Pecis            1979.0  42.0   106.9   76.3   \n",
       "2          Avery Singer            1987.0  34.0    76.2   98.5   \n",
       "3      Flora Yukhnovich            1990.0  31.0   169.5  220.0   \n",
       "4           Salman Toor            1983.0  38.0    99.5   80.0   \n",
       "...                 ...               ...   ...     ...    ...   \n",
       "10011         Fred Deux            1924.0  91.0    59.5   83.0   \n",
       "10012         Fred Deux            1924.0  91.0    71.0  101.0   \n",
       "10013         Fred Deux            1924.0  91.0    73.0  102.0   \n",
       "10014         Fred Deux            1924.0  91.0    46.5   17.5   \n",
       "10015         Fred Deux            1924.0  91.0    30.0   47.5   \n",
       "\n",
       "                     title_of_work  height  year_of_work  age_at_work  \\\n",
       "0                         Untitled     1.0        1973.0         48.0   \n",
       "1                     Summer Patio     1.0        2019.0         40.0   \n",
       "2                 Untitled (Study)     1.0        2016.0         29.0   \n",
       "3      I'll Have What She's Having     1.0        2020.0         30.0   \n",
       "4                     Garden Party     1.0        2018.0         35.0   \n",
       "...                            ...     ...           ...          ...   \n",
       "10011              Détail d'infini     1.0        1960.0         36.0   \n",
       "10012                   Sans titre     1.0        1959.0         35.0   \n",
       "10013                     Untitled     1.0        1959.0         35.0   \n",
       "10014           Untitled [3 works]     1.0        1959.0         35.0   \n",
       "10015           Untitled [3 works]     1.0        1959.0         35.0   \n",
       "\n",
       "       lower_est_USD  upper_est_USD  sold_in_USD  Material_Category  act_area  \\\n",
       "0           74930.94       99907.92    440593.93                  1   5265.00   \n",
       "1           62442.45       87419.43    251767.96                  1   8156.47   \n",
       "2          187327.35      249769.80    314709.95                  1   7505.70   \n",
       "3           74930.94       99907.92   2814281.22                  1  37290.00   \n",
       "4          124884.90      187327.35    361916.44                  1   7960.00   \n",
       "...              ...            ...          ...                ...       ...   \n",
       "10011         857.52        1286.28      6752.97                  7   4938.50   \n",
       "10012        1071.90        2143.80      4727.08                  7   7171.00   \n",
       "10013        1071.90        2143.80      6752.97                  7   7446.00   \n",
       "10014        1607.85        2143.80      5672.49                 10    813.75   \n",
       "10015        2143.80        3215.70      6752.97                 10   1425.00   \n",
       "\n",
       "       posthumous_combined  \n",
       "0                        0  \n",
       "1                        0  \n",
       "2                        0  \n",
       "3                        0  \n",
       "4                        0  \n",
       "...                    ...  \n",
       "10011                    1  \n",
       "10012                    1  \n",
       "10013                    1  \n",
       "10014                    1  \n",
       "10015                    1  \n",
       "\n",
       "[4158 rows x 15 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We have the categories and the legend, we do not need the Cleaned_Material column, as we have \"numerified\" it already\n",
    "df.drop('Cleaned_Material', axis=1, inplace=True)\n",
    "df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['artist_name', 'year_artist_born', 'age', 'length', 'width',\n",
       "       'title_of_work', 'height', 'year_of_work', 'age_at_work',\n",
       "       'lower_est_USD', 'upper_est_USD', 'sold_in_USD', 'Material_Category',\n",
       "       'act_area', 'posthumous_combined'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We need to deal with the artist_name column and \"numerify\" it by making a famous score\n",
    "df.to_csv('cleaned_tow.csv', index=False)  # Set index=False to avoid writing row numbers as part of the data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5162, 17)"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#We saved the dataset without the title of work as cleaned.csv and with the title of work as cleaned_tow.csv, \n",
    "#The cleaning and pre-processing is mostly done, we finish that up and move on to training the models in the next notebook\n",
    "\n",
    "df.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}