PreProcessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d129669-183a-4a4c-8500-e5ffcf2ba28f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader, ConcatDataset\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import transforms\n",
    "import os\n",
    "from torch import nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0a5aaab7-967b-4dac-b4d4-b301a6ef98a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e83de9c-3bc9-410a-ac67-f9c19d395ff9",
   "metadata": {},
   "source": [
    "### Data PipeLine 만들기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cabec666-1a5f-4818-8c8a-bd66f84f5133",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>age</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000001</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>45</td>\n",
       "      <td>000001_female_Asian_45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000002</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>52</td>\n",
       "      <td>000002_female_Asian_52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000004</td>\n",
       "      <td>male</td>\n",
       "      <td>Asian</td>\n",
       "      <td>54</td>\n",
       "      <td>000004_male_Asian_54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000005</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>58</td>\n",
       "      <td>000005_female_Asian_58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000006</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>59</td>\n",
       "      <td>000006_female_Asian_59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  gender   race  age                    path\n",
       "0  000001  female  Asian   45  000001_female_Asian_45\n",
       "1  000002  female  Asian   52  000002_female_Asian_52\n",
       "2  000004    male  Asian   54    000004_male_Asian_54\n",
       "3  000005  female  Asian   58  000005_female_Asian_58\n",
       "4  000006  female  Asian   59  000006_female_Asian_59"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"/opt/ml/input/data/train/train.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1f956070-683b-41f6-be31-58cc54423e83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>age</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000001</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>45</td>\n",
       "      <td>000001_female_Asian_45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000002</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>52</td>\n",
       "      <td>000002_female_Asian_52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000004</td>\n",
       "      <td>male</td>\n",
       "      <td>Asian</td>\n",
       "      <td>54</td>\n",
       "      <td>000004_male_Asian_54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000005</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>58</td>\n",
       "      <td>000005_female_Asian_58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000006</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>59</td>\n",
       "      <td>000006_female_Asian_59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  gender   race  age                    path\n",
       "0  000001  female  Asian   45  000001_female_Asian_45\n",
       "1  000002  female  Asian   52  000002_female_Asian_52\n",
       "2  000004    male  Asian   54    000004_male_Asian_54\n",
       "3  000005  female  Asian   58  000005_female_Asian_58\n",
       "4  000006  female  Asian   59  000006_female_Asian_59"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_df = df.copy()\n",
    "age_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea10d695-9cc4-48e6-9ff3-043a04ae0af3",
   "metadata": {},
   "source": [
    "### 53 ~ 57 제거하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "acf18158-12c3-4cd2-857f-30bb9a9a0396",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "      <th>age</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000001</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>45</td>\n",
       "      <td>000001_female_Asian_45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000002</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>52</td>\n",
       "      <td>000002_female_Asian_52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000005</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>58</td>\n",
       "      <td>000005_female_Asian_58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000006</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>59</td>\n",
       "      <td>000006_female_Asian_59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>000007</td>\n",
       "      <td>female</td>\n",
       "      <td>Asian</td>\n",
       "      <td>58</td>\n",
       "      <td>000007_female_Asian_58</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  gender   race  age                    path\n",
       "0  000001  female  Asian   45  000001_female_Asian_45\n",
       "1  000002  female  Asian   52  000002_female_Asian_52\n",
       "3  000005  female  Asian   58  000005_female_Asian_58\n",
       "4  000006  female  Asian   59  000006_female_Asian_59\n",
       "5  000007  female  Asian   58  000007_female_Asian_58"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "age_df = age_df[~age_df['age'].isin([53, 54, 55, 56, 57])]\n",
    "age_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d70c8380-21b4-4ca9-a065-1496fe9fac2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c6120ae-74b3-43e7-9f2e-9c739da6b388",
   "metadata": {},
   "source": [
    "### image 경로와 label 붙이기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "29c63518-0ee1-4eea-b290-de40d170c84d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18897</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18898</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18899</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Id                                               path label\n",
       "18897  006959  /opt/ml/input/data/train/images/006959_male_As...     0\n",
       "18898  006959  /opt/ml/input/data/train/images/006959_male_As...     0\n",
       "18899  006959  /opt/ml/input/data/train/images/006959_male_As...     0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "root = '/opt/ml/input/data/train/images/'\n",
    "data_list = pd.DataFrame(None, columns = ['Id', 'path', 'label'])\n",
    "for path in df['path']:\n",
    "    Id, gender, race, age = path.split(\"_\")\n",
    "    age = int(age)\n",
    "    age_point = 0\n",
    "    if age < 30:\n",
    "        age_point = 0\n",
    "    elif age < 58:\n",
    "        age_point = 1\n",
    "    else:\n",
    "        age_point = 2\n",
    "        \n",
    "    gender_point = 0\n",
    "    if gender == \"male\":\n",
    "        gender_point = 0\n",
    "    else:\n",
    "        gender_point = 3\n",
    "        \n",
    "    for image in list(os.listdir(os.path.join(root, path))):\n",
    "        if image[0] == \".\":\n",
    "            continue\n",
    "        if image.split('.')[0] == 'normal':\n",
    "            label = 12 + age_point + gender_point\n",
    "            data = {'Id' : Id ,'path' : os.path.join(f'{root}',  path, f'{image}'), 'label' : label}\n",
    "            data_list = data_list.append(data, ignore_index = True)\n",
    "        elif image.split('.')[0] == 'incorrect_mask':\n",
    "            label = 6 + age_point + gender_point\n",
    "            data = {'Id' : Id ,'path' : os.path.join(f'{root}', path, f'{image}'), 'label' : label}\n",
    "            data_list = data_list.append(data, ignore_index = True)\n",
    "        elif 'mask' in image.split('.')[0]:\n",
    "            label = age_point + gender_point\n",
    "            data = {'Id' : Id ,'path' : os.path.join(f'{root}',  path, f'{image}'), 'label' : label}\n",
    "            data_list = data_list.append(data, ignore_index = True)\n",
    "            \n",
    "data_list.tail(3)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07e4442c-f894-411f-93cc-230758b2c931",
   "metadata": {},
   "source": [
    "### Mask, Gender, Age 별로 label 붙이기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "204ad3b8-78a1-476d-8afb-680139dd31bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "mask_dict = {}\n",
    "gender_dict = {}\n",
    "age_dict = {}\n",
    "for i in range(18):\n",
    "    if i % 3 == 0:\n",
    "        age_dict[i] = 0\n",
    "    elif i % 3 == 1:\n",
    "        age_dict[i] = 1\n",
    "    else:\n",
    "        age_dict[i] = 2\n",
    "        \n",
    "\n",
    "for i in range(18):\n",
    "    if i <= 5:\n",
    "        mask_dict[i] = 0\n",
    "    elif i <= 11:\n",
    "        mask_dict[i] = 1\n",
    "    else:\n",
    "        mask_dict[i] = 2\n",
    "\n",
    "\n",
    "for i in range(18):\n",
    "    if i <= 2 or (6 <= i <= 8) or (12 <= i <= 14):\n",
    "        gender_dict[i] = 0\n",
    "    else:\n",
    "        gender_dict[i] = 1\n",
    "data_list['Gender'] = data_list['label'].map(gender_dict)\n",
    "data_list['Mask'] = data_list['label'].map(mask_dict)\n",
    "data_list['Age'] = data_list['label'].map(age_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ce6c965b-28cf-426d-ae41-a6ac91fe43ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Mask</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18897</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18898</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18899</th>\n",
       "      <td>006959</td>\n",
       "      <td>/opt/ml/input/data/train/images/006959_male_As...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Id                                               path label  \\\n",
       "18897  006959  /opt/ml/input/data/train/images/006959_male_As...     0   \n",
       "18898  006959  /opt/ml/input/data/train/images/006959_male_As...     0   \n",
       "18899  006959  /opt/ml/input/data/train/images/006959_male_As...     0   \n",
       "\n",
       "       Gender  Mask  Age  \n",
       "18897       0     0    0  \n",
       "18898       0     0    0  \n",
       "18899       0     0    0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_list.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d70aa2ce-c3f9-4404-8f87-04b43d0ae5b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_list.ifn"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}